diff --git a/wandb/run-20240803_191521-sg37tylz/files/config.yaml b/wandb/run-20240803_191521-sg37tylz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65b2867ec108f6602a8acf4dab0177933b988fe9 --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-mistral-sample_train_2024-08-03-19:14:48 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +save: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +base_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1600 +micro_batch_size: + desc: null + value: 40 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-mistral-sample +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32768 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722680121.573481 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 256 +model_type: + desc: null + value: mistral +max_position_embeddings: + desc: null + value: 512 +num_attention_heads: + desc: null + value: 4 +num_hidden_layers: + desc: null + value: 4 +model_architecture: + desc: null + value: MistralForCausalLM diff --git a/wandb/run-20240803_191521-sg37tylz/files/output.log b/wandb/run-20240803_191521-sg37tylz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..05d630b319de9f57fc9bb543543209f58d20f565 --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/files/output.log @@ -0,0 +1,103 @@ +Created Hugging Face repository with ID koichi12/tiny-mistral-sample. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Loading model state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +Loaded model state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +--> Model /share/pretrained_lm/custom/tiny-mistral +--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 32000000 + validation: 1616000 + test: 16000 +> building train, validation, and test datasets for GPT ... +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +Loaded optimizer state dict from /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): MistralForCausalLM( + (model): MistralModel( + (embed_tokens): Embedding(32768, 256) + (layers): ModuleList( + (0-3): 4 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): MistralDecoderLayer( + (self_attn): MistralFlashAttention2( + (q_proj): Linear(in_features=256, out_features=512, bias=False) + (k_proj): Linear(in_features=256, out_features=256, bias=False) + (v_proj): Linear(in_features=256, out_features=256, bias=False) + (o_proj): Linear(in_features=512, out_features=256, bias=False) + (rotary_emb): MistralRotaryEmbedding() + ) + (mlp): MistralMLP( + (gate_proj): Linear(in_features=256, out_features=512, bias=False) + (up_proj): Linear(in_features=256, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=256, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): MistralRMSNorm() + (post_attention_layernorm): MistralRMSNorm() + ) + ) + ) + ) + (norm): MistralRMSNorm() + ) + (lm_head): Linear(in_features=256, out_features=32768, bias=False) + ) +) +model config: MistralConfig { + "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 256, + "initializer_range": 0.02, + "intermediate_size": 512, + "label_smoothing": 0.0, + "max_position_embeddings": 512, + "model_type": "mistral", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32768 +} +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0020000, took 0.18s +[rank0]:[2024-08-03 19:15:37,067] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +[rank0]:[2024-08-03 19:15:37,199] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010301990000698424, 'preprocessing_with_comm': 0.009796595000352681, 'state_converting': 0.007119276000139507, : 0.018263464000028762}) \ No newline at end of file diff --git a/wandb/run-20240803_191521-sg37tylz/files/requirements.txt b/wandb/run-20240803_191521-sg37tylz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json b/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..20f1702b00260f23f917f1b75aa0cc575792720e --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-03T10:15:22.462858", + "startedAt": "2024-08-03T10:15:21.560082", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "40", + "--global-batch-size", + "1600", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/custom/tiny-mistral", + "--save", + "/work/llm_recipes/models/tiny-mistral-sample", + "--load", + "/work/llm_recipes/models/tiny-mistral-sample", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-mistral-sample", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-mistral-sample_train_2024-08-03-19:14:48" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json b/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0724bd7df12f46eff44a25860c977ed8e96c2574 --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 14}} \ No newline at end of file diff --git a/wandb/run-20240803_191521-sg37tylz/logs/debug-internal.log b/wandb/run-20240803_191521-sg37tylz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..11dec0c956e499d94a7e5fa37eab46ede7271729 --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/logs/debug-internal.log @@ -0,0 +1,194 @@ +2024-08-03 19:15:21,574 INFO StreamThr :9246 [internal.py:wandb_internal():86] W&B internal server running at pid: 9246, started at: 2024-08-03 19:15:21.573011 +2024-08-03 19:15:21,575 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: status +2024-08-03 19:15:21,578 INFO WriterThread:9246 [datastore.py:open_for_write():87] open: /project/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb +2024-08-03 19:15:21,579 DEBUG SenderThread:9246 [sender.py:send():382] send: header +2024-08-03 19:15:21,755 DEBUG SenderThread:9246 [sender.py:send():382] send: run +2024-08-03 19:15:22,256 INFO SenderThread:9246 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240803_191521-sg37tylz/files +2024-08-03 19:15:22,257 INFO SenderThread:9246 [sender.py:_start_run_threads():1136] run started: sg37tylz with start time 1722680121.573481 +2024-08-03 19:15:22,262 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: check_version +2024-08-03 19:15:22,262 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: check_version +2024-08-03 19:15:22,442 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: run_start +2024-08-03 19:15:22,448 DEBUG HandlerThread:9246 [system_info.py:__init__():27] System info init +2024-08-03 19:15:22,448 DEBUG HandlerThread:9246 [system_info.py:__init__():42] System info init done +2024-08-03 19:15:22,448 INFO HandlerThread:9246 [system_monitor.py:start():194] Starting system monitor +2024-08-03 19:15:22,448 INFO SystemMonitor:9246 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-03 19:15:22,449 INFO HandlerThread:9246 [system_monitor.py:probe():214] Collecting system info +2024-08-03 19:15:22,449 INFO SystemMonitor:9246 [interfaces.py:start():190] Started cpu monitoring +2024-08-03 19:15:22,450 INFO SystemMonitor:9246 [interfaces.py:start():190] Started disk monitoring +2024-08-03 19:15:22,450 INFO SystemMonitor:9246 [interfaces.py:start():190] Started gpu monitoring +2024-08-03 19:15:22,451 INFO SystemMonitor:9246 [interfaces.py:start():190] Started memory monitoring +2024-08-03 19:15:22,452 INFO SystemMonitor:9246 [interfaces.py:start():190] Started network monitoring +2024-08-03 19:15:22,462 DEBUG HandlerThread:9246 [system_info.py:probe():151] Probing system +2024-08-03 19:15:22,464 DEBUG HandlerThread:9246 [system_info.py:_probe_git():136] Probing git +2024-08-03 19:15:22,476 DEBUG HandlerThread:9246 [system_info.py:_probe_git():144] Probing git done +2024-08-03 19:15:22,476 DEBUG HandlerThread:9246 [system_info.py:probe():199] Probing system done +2024-08-03 19:15:22,476 DEBUG HandlerThread:9246 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T10:15:22.462858', 'startedAt': '2024-08-03T10:15:21.560082', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '40', '--global-batch-size', '1600', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample', '--load', '/work/llm_recipes/models/tiny-mistral-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample_train_2024-08-03-19:14:48'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-03 19:15:22,476 INFO HandlerThread:9246 [system_monitor.py:probe():224] Finished collecting system info +2024-08-03 19:15:22,476 INFO HandlerThread:9246 [system_monitor.py:probe():227] Publishing system info +2024-08-03 19:15:22,478 INFO HandlerThread:9246 [system_monitor.py:probe():229] Finished publishing system info +2024-08-03 19:15:22,505 DEBUG SenderThread:9246 [sender.py:send():382] send: files +2024-08-03 19:15:22,505 INFO SenderThread:9246 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-03 19:15:22,514 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: python_packages +2024-08-03 19:15:22,514 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:15:22,514 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:15:22,514 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: python_packages +2024-08-03 19:15:22,527 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:15:22,755 DEBUG SenderThread:9246 [sender.py:send():382] send: telemetry +2024-08-03 19:15:23,258 INFO Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/requirements.txt +2024-08-03 19:15:23,258 INFO Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json +2024-08-03 19:15:25,596 INFO wandb-upload_0:9246 [upload_job.py:push():131] Uploaded file /tmp/tmp1lfwq_epwandb/7v7ji8nj-wandb-metadata.json +2024-08-03 19:15:26,756 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:15:28,261 INFO Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:30,262 INFO Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:31,262 INFO Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:31,904 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:15:33,263 INFO Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:37,068 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:15:37,110 DEBUG SenderThread:9246 [sender.py:send():382] send: config +2024-08-03 19:15:37,111 DEBUG SenderThread:9246 [sender.py:send():382] send: config +2024-08-03 19:15:37,265 INFO Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:37,316 DEBUG SenderThread:9246 [sender.py:send():382] send: exit +2024-08-03 19:15:37,316 INFO SenderThread:9246 [sender.py:send_exit():589] handling exit code: 0 +2024-08-03 19:15:37,317 INFO SenderThread:9246 [sender.py:send_exit():591] handling runtime: 14 +2024-08-03 19:15:37,318 INFO SenderThread:9246 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:15:37,318 INFO SenderThread:9246 [sender.py:send_exit():597] send defer +2024-08-03 19:15:37,318 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,318 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-03 19:15:37,318 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,318 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-03 19:15:37,318 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 1 +2024-08-03 19:15:37,319 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,319 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-03 19:15:37,319 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,319 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-03 19:15:37,319 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 2 +2024-08-03 19:15:37,319 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,319 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-03 19:15:37,319 INFO HandlerThread:9246 [system_monitor.py:finish():203] Stopping system monitor +2024-08-03 19:15:37,319 DEBUG SystemMonitor:9246 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-03 19:15:37,319 INFO HandlerThread:9246 [interfaces.py:finish():202] Joined cpu monitor +2024-08-03 19:15:37,320 DEBUG SystemMonitor:9246 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-03 19:15:37,320 INFO HandlerThread:9246 [interfaces.py:finish():202] Joined disk monitor +2024-08-03 19:15:37,320 DEBUG SystemMonitor:9246 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-03 19:15:37,351 INFO HandlerThread:9246 [interfaces.py:finish():202] Joined gpu monitor +2024-08-03 19:15:37,351 INFO HandlerThread:9246 [interfaces.py:finish():202] Joined memory monitor +2024-08-03 19:15:37,351 INFO HandlerThread:9246 [interfaces.py:finish():202] Joined network monitor +2024-08-03 19:15:37,352 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,352 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-03 19:15:37,352 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 3 +2024-08-03 19:15:37,352 DEBUG SenderThread:9246 [sender.py:send():382] send: stats +2024-08-03 19:15:37,352 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,352 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-03 19:15:37,353 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,353 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-03 19:15:37,353 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 4 +2024-08-03 19:15:37,353 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,353 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-03 19:15:37,353 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,353 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-03 19:15:37,353 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 5 +2024-08-03 19:15:37,353 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,353 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-03 19:15:37,353 DEBUG SenderThread:9246 [sender.py:send():382] send: summary +2024-08-03 19:15:37,354 INFO SenderThread:9246 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:15:37,354 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,354 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-03 19:15:37,354 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 6 +2024-08-03 19:15:37,355 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,355 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-03 19:15:37,355 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,355 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-03 19:15:37,357 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:15:37,549 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 7 +2024-08-03 19:15:37,550 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:37,550 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-03 19:15:37,550 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:37,550 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-03 19:15:38,267 INFO Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/config.yaml +2024-08-03 19:15:38,267 INFO Thread-12 :9246 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json +2024-08-03 19:15:38,316 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:15:39,265 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 8 +2024-08-03 19:15:39,265 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:15:39,265 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:39,265 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-03 19:15:39,265 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:39,265 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-03 19:15:39,266 INFO SenderThread:9246 [job_builder.py:build():296] Attempting to build job artifact +2024-08-03 19:15:39,266 INFO SenderThread:9246 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-03 19:15:39,267 INFO Thread-12 :9246 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:39,281 INFO SenderThread:9246 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-03 19:15:39,289 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 9 +2024-08-03 19:15:39,290 DEBUG SenderThread:9246 [sender.py:send():382] send: artifact +2024-08-03 19:15:39,290 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:39,291 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-03 19:15:39,317 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:15:40,500 INFO wandb-upload_1:9246 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmpftiijukc +2024-08-03 19:15:40,873 INFO wandb-upload_0:9246 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp8adahfv9 +2024-08-03 19:15:42,354 INFO SenderThread:9246 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA4NTUyMzkyNA==', 'versionIndex': 0}}} +2024-08-03 19:15:42,354 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:42,354 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-03 19:15:42,354 INFO SenderThread:9246 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-03 19:15:43,268 INFO SenderThread:9246 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240803_191521-sg37tylz/files +2024-08-03 19:15:43,268 INFO SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/requirements.txt requirements.txt +2024-08-03 19:15:43,269 INFO SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/config.yaml config.yaml +2024-08-03 19:15:43,270 INFO SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-metadata.json wandb-metadata.json +2024-08-03 19:15:43,270 INFO SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json wandb-summary.json +2024-08-03 19:15:43,271 INFO SenderThread:9246 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191521-sg37tylz/files/output.log output.log +2024-08-03 19:15:43,273 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 10 +2024-08-03 19:15:43,273 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:15:43,273 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:43,275 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-03 19:15:43,275 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:43,275 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-03 19:15:43,275 INFO SenderThread:9246 [file_pusher.py:finish():172] shutting down file pusher +2024-08-03 19:15:43,318 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:15:43,318 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:15:43,685 INFO wandb-upload_1:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/requirements.txt +2024-08-03 19:15:43,839 INFO wandb-upload_0:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/config.yaml +2024-08-03 19:15:43,870 INFO wandb-upload_2:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/wandb-summary.json +2024-08-03 19:15:43,873 INFO wandb-upload_3:9246 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191521-sg37tylz/files/output.log +2024-08-03 19:15:44,073 INFO Thread-11 (_thread_body):9246 [sender.py:transition_state():617] send defer: 11 +2024-08-03 19:15:44,074 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:44,074 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-03 19:15:44,074 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:44,074 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-03 19:15:44,074 INFO SenderThread:9246 [file_pusher.py:join():178] waiting for file pusher +2024-08-03 19:15:44,074 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 12 +2024-08-03 19:15:44,074 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:44,074 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-03 19:15:44,074 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:44,074 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-03 19:15:44,074 INFO SenderThread:9246 [file_stream.py:finish():595] file stream finish called +2024-08-03 19:15:44,248 INFO SenderThread:9246 [file_stream.py:finish():599] file stream finish is done +2024-08-03 19:15:44,248 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 13 +2024-08-03 19:15:44,248 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:44,248 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-03 19:15:44,248 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:44,248 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-03 19:15:44,249 INFO SenderThread:9246 [sender.py:transition_state():617] send defer: 14 +2024-08-03 19:15:44,249 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:15:44,249 DEBUG SenderThread:9246 [sender.py:send():382] send: final +2024-08-03 19:15:44,249 INFO HandlerThread:9246 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-03 19:15:44,249 DEBUG SenderThread:9246 [sender.py:send():382] send: footer +2024-08-03 19:15:44,249 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: defer +2024-08-03 19:15:44,249 INFO SenderThread:9246 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-03 19:15:44,250 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:15:44,250 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:15:44,250 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:15:44,250 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: server_info +2024-08-03 19:15:44,251 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: get_summary +2024-08-03 19:15:44,251 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:15:44,251 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: server_info +2024-08-03 19:15:44,252 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-03 19:15:44,252 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:15:44,253 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: job_info +2024-08-03 19:15:44,413 DEBUG SenderThread:9246 [sender.py:send_request():409] send_request: job_info +2024-08-03 19:15:44,413 INFO MainThread:9246 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-03 19:15:44,413 INFO MainThread:9246 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-03 19:15:44,414 INFO MainThread:9246 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-03 19:15:44,414 DEBUG HandlerThread:9246 [handler.py:handle_request():146] handle_request: shutdown +2024-08-03 19:15:44,414 INFO HandlerThread:9246 [handler.py:finish():869] shutting down handler +2024-08-03 19:15:45,253 INFO WriterThread:9246 [datastore.py:close():296] close: /project/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb +2024-08-03 19:15:45,413 INFO SenderThread:9246 [sender.py:finish():1572] shutting down sender +2024-08-03 19:15:45,413 INFO SenderThread:9246 [file_pusher.py:finish():172] shutting down file pusher +2024-08-03 19:15:45,414 INFO SenderThread:9246 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240803_191521-sg37tylz/logs/debug.log b/wandb/run-20240803_191521-sg37tylz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3662a63282e27accdb756beb5b7eacbfbe72a0d8 --- /dev/null +++ b/wandb/run-20240803_191521-sg37tylz/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-03 19:15:21,565 INFO MainThread:9173 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_setup.py:_flush():76] Configure stats pid to 9173 +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240803_191521-sg37tylz/logs/debug.log +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240803_191521-sg37tylz/logs/debug-internal.log +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_init.py:init():566] calling init triggers +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample_train_2024-08-03-19:14:48', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample', 'save': '/work/llm_recipes/models/tiny-mistral-sample', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 1600, 'micro_batch_size': 40, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40} +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_init.py:init():616] starting backend +2024-08-03 19:15:21,566 INFO MainThread:9173 [wandb_init.py:init():620] setting up manager +2024-08-03 19:15:21,571 INFO MainThread:9173 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-03 19:15:21,573 INFO MainThread:9173 [wandb_init.py:init():628] backend started and connected +2024-08-03 19:15:21,578 INFO MainThread:9173 [wandb_init.py:init():720] updated telemetry +2024-08-03 19:15:21,751 INFO MainThread:9173 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-03 19:15:22,261 INFO MainThread:9173 [wandb_run.py:_on_init():2262] communicating current version +2024-08-03 19:15:22,435 INFO MainThread:9173 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-03 19:15:22,435 INFO MainThread:9173 [wandb_init.py:init():804] starting run threads in backend +2024-08-03 19:15:22,513 INFO MainThread:9173 [wandb_run.py:_console_start():2241] atexit reg +2024-08-03 19:15:22,514 INFO MainThread:9173 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-03 19:15:22,514 INFO MainThread:9173 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-03 19:15:22,514 INFO MainThread:9173 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-03 19:15:22,514 INFO MainThread:9173 [wandb_init.py:init():847] run started, returning control to user process +2024-08-03 19:15:37,109 INFO MainThread:9173 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'} +2024-08-03 19:15:37,110 INFO MainThread:9173 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-03 19:15:45,414 WARNING MsgRouterThr:9173 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb b/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bd03979bca0d05b8e6b1a4fa14b22196da588662 Binary files /dev/null and b/wandb/run-20240803_191521-sg37tylz/run-sg37tylz.wandb differ diff --git a/wandb/run-20240803_191815-jdwps0z3/files/config.yaml b/wandb/run-20240803_191815-jdwps0z3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d76d2245cf46e499e9d39ca5b635b8a35a91fc4 --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-mistral-sample2_train_2024-08-03-19:18:05 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample2 +save: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample2 +base_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1600 +micro_batch_size: + desc: null + value: 40 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-mistral-sample2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32768 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722680295.872336 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 256 +model_type: + desc: null + value: mistral +max_position_embeddings: + desc: null + value: 512 +num_attention_heads: + desc: null + value: 4 +num_hidden_layers: + desc: null + value: 4 +model_architecture: + desc: null + value: MistralForCausalLM diff --git a/wandb/run-20240803_191815-jdwps0z3/files/output.log b/wandb/run-20240803_191815-jdwps0z3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..79573891e025f47b142461e54106f753fe481196 --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/files/output.log @@ -0,0 +1,239 @@ +Created Hugging Face repository with ID koichi12/tiny-mistral-sample2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample2, skipping model loading +--> Model /share/pretrained_lm/custom/tiny-mistral +--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 32000000 + validation: 1616000 + test: 16000 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample2, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): MistralForCausalLM( + (model): MistralModel( + (embed_tokens): Embedding(32768, 256) + (layers): ModuleList( + (0-3): 4 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): MistralDecoderLayer( + (self_attn): MistralFlashAttention2( + (q_proj): Linear(in_features=256, out_features=512, bias=False) + (k_proj): Linear(in_features=256, out_features=256, bias=False) + (v_proj): Linear(in_features=256, out_features=256, bias=False) + (o_proj): Linear(in_features=512, out_features=256, bias=False) + (rotary_emb): MistralRotaryEmbedding() + ) + (mlp): MistralMLP( + (gate_proj): Linear(in_features=256, out_features=512, bias=False) + (up_proj): Linear(in_features=256, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=256, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): MistralRMSNorm() + (post_attention_layernorm): MistralRMSNorm() + ) + ) + ) + ) + (norm): MistralRMSNorm() + ) + (lm_head): Linear(in_features=256, out_features=32768, bias=False) + ) +) +model config: MistralConfig { + "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 256, + "initializer_range": 0.02, + "intermediate_size": 512, + "label_smoothing": 0.0, + "max_position_embeddings": 512, + "model_type": "mistral", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32768 +} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 7.474408749953364, Tokens per sec: 105583.91620199519, Loss: 10.45809555053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 19.626322889381036, Tokens per sec: 277242.53524650185, Loss: 10.456090927124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 19.74558354343518, Tokens per sec: 278927.2179184158, Loss: 10.458115577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 19.588618586475842, Tokens per sec: 276709.92215407215, Loss: 10.456512451171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 19.72619917981179, Tokens per sec: 278653.39331329847, Loss: 10.458467483520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 19.627595003583973, Tokens per sec: 277260.5052029086, Loss: 10.457172393798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 19.77370988376223, Tokens per sec: 279324.53217557067, Loss: 10.457324981689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 19.77766998452317, Tokens per sec: 279380.47278049105, Loss: 10.457306861877441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 19.749958254856136, Tokens per sec: 278989.01533671736, Loss: 10.457944869995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 19.752309517490676, Tokens per sec: 279022.22940424824, Loss: 10.45663833618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 19.74392548352723, Tokens per sec: 278903.7960713861, Loss: 10.456352233886719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 19.758394357809003, Tokens per sec: 279108.18419903744, Loss: 10.455950736999512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 19.729624964309398, Tokens per sec: 278701.78613678756, Loss: 10.45804214477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 19.626730348672773, Tokens per sec: 277248.29103925463, Loss: 10.457955360412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 19.724026466287597, Tokens per sec: 278622.7014404904, Loss: 10.459123611450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 19.108803574684035, Tokens per sec: 269932.0284514028, Loss: 10.45695686340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 19.73958477407385, Tokens per sec: 278842.4789667809, Loss: 10.456724166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 19.47597897335848, Tokens per sec: 275118.76867688465, Loss: 10.456052780151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 19.72614156699911, Tokens per sec: 278652.57947148307, Loss: 10.458043098449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 19.701494428407866, Tokens per sec: 278304.4126127127, Loss: 10.455198287963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 19.74596953791029, Tokens per sec: 278932.6704979679, Loss: 10.457011222839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 19.615963035173014, Tokens per sec: 277096.1913663158, Loss: 10.457304954528809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 19.718852381531324, Tokens per sec: 278549.61202972836, Loss: 10.456293106079102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 19.67743842750437, Tokens per sec: 277964.5961979942, Loss: 10.456793785095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 19.72792104268366, Tokens per sec: 278677.7164445664, Loss: 10.45568561553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 19.766049190998213, Tokens per sec: 279216.31680096325, Loss: 10.455270767211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 19.748005203155174, Tokens per sec: 278961.4264191145, Loss: 10.456525802612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 19.788746629218007, Tokens per sec: 279536.9420831989, Loss: 10.458827018737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 19.64595613343327, Tokens per sec: 277519.8755504821, Loss: 10.454755783081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 19.791751212574006, Tokens per sec: 279579.38499579503, Loss: 10.455424308776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 19.647830180890995, Tokens per sec: 277546.34844972467, Loss: 10.455726623535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 19.108735127334096, Tokens per sec: 269931.06156030786, Loss: 10.456134796142578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 19.790956814139804, Tokens per sec: 279568.16327906615, Loss: 10.45483684539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 19.809776481063633, Tokens per sec: 279834.0109470101, Loss: 10.455580711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 19.78921404344017, Tokens per sec: 279543.5448026535, Loss: 10.4553861618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 19.760295167208643, Tokens per sec: 279135.03513896884, Loss: 10.455459594726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 19.77547756611738, Tokens per sec: 279349.502555423, Loss: 10.45541000366211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 19.777886920468827, Tokens per sec: 279383.53722979716, Loss: 10.455677032470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 19.686203609180314, Tokens per sec: 278088.4136447687, Loss: 10.454801559448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 19.733730730492624, Tokens per sec: 278759.7844196133, Loss: 10.45442008972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 19.781936717382273, Tokens per sec: 279440.74488758645, Loss: 10.453974723815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 19.713193575703478, Tokens per sec: 278469.67542198877, Loss: 10.454327583312988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 19.765683899965286, Tokens per sec: 279211.1566793938, Loss: 10.453243255615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 19.737930705062368, Tokens per sec: 278819.11349537794, Loss: 10.452882766723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 19.768669599163157, Tokens per sec: 279253.3328333156, Loss: 10.452847480773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 19.195477136883916, Tokens per sec: 271156.3840405051, Loss: 10.452914237976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 19.78773782868223, Tokens per sec: 279522.69171038724, Loss: 10.452442169189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 19.79785807451266, Tokens per sec: 279665.6508692251, Loss: 10.452777862548828 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt b/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json b/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cef233425de84d18c18c64b8e4397f7e2e22fad8 --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-03T10:18:16.495932", + "startedAt": "2024-08-03T10:18:15.860061", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "40", + "--global-batch-size", + "1600", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/custom/tiny-mistral", + "--save", + "/work/llm_recipes/models/tiny-mistral-sample2", + "--load", + "/work/llm_recipes/models/tiny-mistral-sample2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-mistral-sample2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-mistral-sample2_train_2024-08-03-19:18:05" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json b/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..646f6ae049a990c10b2fcd88bfed4651620ab11a --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 10.452777862548828, "training/perplexity": 34640.467638738424, "utils/batch_size": 40, "utils/global_batch_size": 1600, "utils/seq_len": 513, "utils/gradient_accumulation_steps": 40, "utils/iteration": 48, "optimizer/lr": 2.8240000000000004e-06, "optimizer/variance_l2": 0.0037958921404718368, "optimizer/variance_sqrt_l2": 0.9574357404136762, "optimizer/momentum_l2": 0.9945472829864366, "optimizer/weight_l2": 101.38293742045552, "optimizer/variance_l1": 0.91748046875, "optimizer/variance_sqrt_l1": 723.0, "optimizer/momentum_l1": 740.5, "optimizer/weight_l1": 320512.0, "optimizer/variance_abs_max": 0.0003185272216796875, "optimizer/variance_sqrt_abs_max": 0.017822265625, "optimizer/momentum_abs_max": 0.0184326171875, "optimizer/weight_abs_max": 1.0, "stats/1_iteration_time": 2.9349331869998423, "stats/tokens_per_sec": 279665.6508692251, "stats/tokens_per_sec_per_gpu": 279665.6508692251, "stats/tflops": 19.79785807451266, "_timestamp": 1722680446.903461, "_runtime": 151.03112506866455, "_step": 48, "_wandb": {"runtime": 151}} \ No newline at end of file diff --git a/wandb/run-20240803_191815-jdwps0z3/logs/debug-internal.log b/wandb/run-20240803_191815-jdwps0z3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..fb8134324c2036e57da7d696facc020a4f6ce04f --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/logs/debug-internal.log @@ -0,0 +1,524 @@ +2024-08-03 19:18:15,873 INFO StreamThr :9504 [internal.py:wandb_internal():86] W&B internal server running at pid: 9504, started at: 2024-08-03 19:18:15.872881 +2024-08-03 19:18:15,875 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status +2024-08-03 19:18:15,877 INFO WriterThread:9504 [datastore.py:open_for_write():87] open: /project/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb +2024-08-03 19:18:15,878 DEBUG SenderThread:9504 [sender.py:send():382] send: header +2024-08-03 19:18:15,892 DEBUG SenderThread:9504 [sender.py:send():382] send: run +2024-08-03 19:18:16,382 INFO SenderThread:9504 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240803_191815-jdwps0z3/files +2024-08-03 19:18:16,382 INFO SenderThread:9504 [sender.py:_start_run_threads():1136] run started: jdwps0z3 with start time 1722680295.872336 +2024-08-03 19:18:16,387 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: check_version +2024-08-03 19:18:16,387 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: check_version +2024-08-03 19:18:16,477 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: run_start +2024-08-03 19:18:16,483 DEBUG HandlerThread:9504 [system_info.py:__init__():27] System info init +2024-08-03 19:18:16,483 DEBUG HandlerThread:9504 [system_info.py:__init__():42] System info init done +2024-08-03 19:18:16,483 INFO HandlerThread:9504 [system_monitor.py:start():194] Starting system monitor +2024-08-03 19:18:16,483 INFO SystemMonitor:9504 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-03 19:18:16,483 INFO HandlerThread:9504 [system_monitor.py:probe():214] Collecting system info +2024-08-03 19:18:16,484 INFO SystemMonitor:9504 [interfaces.py:start():190] Started cpu monitoring +2024-08-03 19:18:16,485 INFO SystemMonitor:9504 [interfaces.py:start():190] Started disk monitoring +2024-08-03 19:18:16,486 INFO SystemMonitor:9504 [interfaces.py:start():190] Started gpu monitoring +2024-08-03 19:18:16,486 INFO SystemMonitor:9504 [interfaces.py:start():190] Started memory monitoring +2024-08-03 19:18:16,487 INFO SystemMonitor:9504 [interfaces.py:start():190] Started network monitoring +2024-08-03 19:18:16,495 DEBUG HandlerThread:9504 [system_info.py:probe():151] Probing system +2024-08-03 19:18:16,497 DEBUG HandlerThread:9504 [system_info.py:_probe_git():136] Probing git +2024-08-03 19:18:16,509 DEBUG HandlerThread:9504 [system_info.py:_probe_git():144] Probing git done +2024-08-03 19:18:16,509 DEBUG HandlerThread:9504 [system_info.py:probe():199] Probing system done +2024-08-03 19:18:16,509 DEBUG HandlerThread:9504 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T10:18:16.495932', 'startedAt': '2024-08-03T10:18:15.860061', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '40', '--global-batch-size', '1600', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample2', '--load', '/work/llm_recipes/models/tiny-mistral-sample2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample2_train_2024-08-03-19:18:05'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-03 19:18:16,509 INFO HandlerThread:9504 [system_monitor.py:probe():224] Finished collecting system info +2024-08-03 19:18:16,509 INFO HandlerThread:9504 [system_monitor.py:probe():227] Publishing system info +2024-08-03 19:18:16,510 INFO HandlerThread:9504 [system_monitor.py:probe():229] Finished publishing system info +2024-08-03 19:18:16,516 DEBUG SenderThread:9504 [sender.py:send():382] send: files +2024-08-03 19:18:16,516 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-03 19:18:16,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: python_packages +2024-08-03 19:18:16,526 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:18:16,526 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:18:16,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: python_packages +2024-08-03 19:18:16,528 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:18:16,829 DEBUG SenderThread:9504 [sender.py:send():382] send: telemetry +2024-08-03 19:18:17,199 INFO wandb-upload_0:9504 [upload_job.py:push():131] Uploaded file /tmp/tmp76a9qs5lwandb/q74vxtjq-wandb-metadata.json +2024-08-03 19:18:17,385 INFO Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json +2024-08-03 19:18:17,386 INFO Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt +2024-08-03 19:18:17,386 INFO Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:17,990 DEBUG SenderThread:9504 [sender.py:send():382] send: config +2024-08-03 19:18:17,991 DEBUG SenderThread:9504 [sender.py:send():382] send: config +2024-08-03 19:18:19,388 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:20,991 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:25,992 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:28,096 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:30,394 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:31,059 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:31,062 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:31,063 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:31,063 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:31,064 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:31,395 INFO Thread-12 :9504 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:31,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:18:31,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:18:31,568 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:18:34,003 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:34,005 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:34,005 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:34,006 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:34,397 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:34,397 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:36,398 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:36,971 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:36,974 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:36,975 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:36,976 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:36,977 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:37,399 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:39,919 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:39,921 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:39,921 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:39,922 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:40,401 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:40,401 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:42,402 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:42,881 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:42,883 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:42,883 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:42,884 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:42,885 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:43,403 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:44,403 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:45,821 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:45,823 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:45,824 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:45,825 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:46,405 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:46,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:18:46,525 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:18:46,526 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:18:48,406 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:48,697 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:48,761 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:48,885 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:48,885 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:48,887 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:49,407 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:49,407 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/config.yaml +2024-08-03 19:18:50,407 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:51,705 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:51,707 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:51,708 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:51,709 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:52,409 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:54,410 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:54,648 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:54,651 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:54,651 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:54,651 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:18:54,652 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:55,411 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:18:56,412 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:18:57,593 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:18:57,596 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:18:57,596 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:18:57,597 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:18:58,413 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:00,414 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:00,536 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:00,540 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:00,540 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:00,541 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:00,542 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:01,415 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:01,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:19:01,525 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:19:01,526 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:19:02,416 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:03,483 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:03,484 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:03,485 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:03,486 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:04,417 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:06,418 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:06,446 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:06,448 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:06,448 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:06,448 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:06,449 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:07,420 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:08,420 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:09,393 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:09,396 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:09,396 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:09,397 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:09,421 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:10,422 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:12,398 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:12,436 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:12,438 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:12,438 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:12,440 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:13,424 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:14,424 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:15,381 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:15,383 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:15,384 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:15,385 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:15,425 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:16,426 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:16,487 DEBUG SystemMonitor:9504 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-03 19:19:16,489 DEBUG SenderThread:9504 [sender.py:send():382] send: stats +2024-08-03 19:19:16,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:19:16,525 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:19:16,526 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:19:17,757 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:18,366 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:18,368 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:18,368 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:18,369 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:18,427 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:20,428 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:21,314 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:21,316 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:21,317 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:21,318 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:21,429 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:22,430 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:23,318 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:24,265 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:24,267 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:24,268 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:24,269 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:24,431 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:26,432 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:27,210 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:27,212 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:27,213 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:27,214 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:27,433 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:28,434 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:29,214 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:30,174 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:30,175 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:30,176 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:30,177 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:30,435 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:31,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:19:31,525 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:19:31,527 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:19:32,436 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:33,125 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:33,126 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:33,126 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:33,127 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:33,437 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:34,438 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:35,169 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:36,079 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:36,081 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:36,082 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:36,084 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:36,439 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:38,441 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:39,026 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:39,029 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:39,029 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:39,030 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:39,442 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:40,443 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:41,031 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:41,968 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:41,970 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:41,970 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:41,972 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:42,444 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:44,445 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:44,912 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:44,914 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:44,915 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:44,916 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:45,447 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:46,447 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:46,489 DEBUG SenderThread:9504 [sender.py:send():382] send: stats +2024-08-03 19:19:46,490 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:46,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:19:46,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:19:46,527 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:19:47,851 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:47,852 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:47,852 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:47,853 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:48,449 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:50,450 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:50,810 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:50,814 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:50,814 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:50,815 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:51,451 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:51,815 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:52,452 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:53,748 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:53,750 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:53,750 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:53,752 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:54,454 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:56,455 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:56,707 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:56,709 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:56,709 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:56,711 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:19:57,456 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:19:57,712 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:19:58,456 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:19:59,749 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:19:59,752 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:19:59,752 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:19:59,754 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:00,458 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:01,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:20:01,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:20:01,527 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:20:02,459 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:02,687 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:02,689 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:02,689 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:02,690 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:02,729 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:03,460 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:04,460 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:05,622 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:05,624 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:05,625 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:05,626 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:06,462 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:08,463 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:08,560 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:08,563 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:08,563 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:08,563 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:08,564 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:09,464 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:10,464 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:11,503 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:11,505 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:11,506 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:11,507 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:12,466 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:12,466 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:14,443 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:14,468 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:14,469 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:14,469 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:14,470 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:15,470 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:16,471 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:16,490 DEBUG SenderThread:9504 [sender.py:send():382] send: stats +2024-08-03 19:20:16,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:20:16,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:20:16,527 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:20:17,382 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:17,384 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:17,384 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:17,385 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:17,471 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:18,472 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:20,336 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:20,338 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:20,339 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:20,339 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:20,340 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:20,473 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:22,474 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:23,282 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:23,284 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:23,285 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:23,286 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:23,475 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:24,476 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:26,221 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:26,223 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:26,224 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:26,224 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:26,225 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:26,477 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:28,478 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:29,171 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:29,173 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:29,174 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:29,175 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:29,479 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:30,480 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:31,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:20:31,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:20:31,527 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:20:31,709 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:32,112 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:32,114 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:32,114 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:32,115 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:32,482 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:34,483 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:35,058 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:35,060 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:35,061 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:35,062 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:35,484 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:36,485 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:37,062 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:37,999 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:38,001 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:38,002 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:38,003 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:38,486 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:40,487 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:41,029 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:41,031 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:41,032 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:41,033 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:41,488 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:42,489 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:43,034 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:43,967 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:43,971 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:43,971 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:43,972 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:44,490 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:46,491 DEBUG SenderThread:9504 [sender.py:send():382] send: stats +2024-08-03 19:20:46,492 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:46,525 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:20:46,526 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:20:46,527 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:20:46,904 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:20:46,905 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:46,906 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:46,907 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:47,493 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:47,496 DEBUG SenderThread:9504 [sender.py:send():382] send: exit +2024-08-03 19:20:47,496 INFO SenderThread:9504 [sender.py:send_exit():589] handling exit code: 255 +2024-08-03 19:20:47,496 INFO SenderThread:9504 [sender.py:send_exit():591] handling runtime: 151 +2024-08-03 19:20:47,497 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:47,497 INFO SenderThread:9504 [sender.py:send_exit():597] send defer +2024-08-03 19:20:47,497 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,497 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-03 19:20:47,497 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,497 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-03 19:20:47,497 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 1 +2024-08-03 19:20:47,498 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,498 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-03 19:20:47,498 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,498 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-03 19:20:47,498 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 2 +2024-08-03 19:20:47,498 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,498 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-03 19:20:47,498 INFO HandlerThread:9504 [system_monitor.py:finish():203] Stopping system monitor +2024-08-03 19:20:47,498 INFO HandlerThread:9504 [interfaces.py:finish():202] Joined cpu monitor +2024-08-03 19:20:47,498 DEBUG SystemMonitor:9504 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-03 19:20:47,499 INFO HandlerThread:9504 [interfaces.py:finish():202] Joined disk monitor +2024-08-03 19:20:47,499 DEBUG SystemMonitor:9504 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-03 19:20:47,532 INFO HandlerThread:9504 [interfaces.py:finish():202] Joined gpu monitor +2024-08-03 19:20:47,532 INFO HandlerThread:9504 [interfaces.py:finish():202] Joined memory monitor +2024-08-03 19:20:47,532 INFO HandlerThread:9504 [interfaces.py:finish():202] Joined network monitor +2024-08-03 19:20:47,533 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,533 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-03 19:20:47,533 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 3 +2024-08-03 19:20:47,533 DEBUG SenderThread:9504 [sender.py:send():382] send: stats +2024-08-03 19:20:47,533 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,533 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-03 19:20:47,536 DEBUG SenderThread:9504 [sender.py:send():382] send: history +2024-08-03 19:20:47,537 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:20:47,537 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:47,538 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,538 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-03 19:20:47,538 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 4 +2024-08-03 19:20:47,538 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,538 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-03 19:20:47,538 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,538 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-03 19:20:47,538 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 5 +2024-08-03 19:20:47,538 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,538 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-03 19:20:47,539 DEBUG SenderThread:9504 [sender.py:send():382] send: summary +2024-08-03 19:20:47,540 INFO SenderThread:9504 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:20:47,540 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,540 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-03 19:20:47,540 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 6 +2024-08-03 19:20:47,541 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,541 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-03 19:20:47,541 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,541 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-03 19:20:47,541 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 7 +2024-08-03 19:20:47,541 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:20:47,541 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,541 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-03 19:20:47,541 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,541 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-03 19:20:47,875 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 8 +2024-08-03 19:20:47,875 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,875 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-03 19:20:47,875 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:47,875 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-03 19:20:47,875 INFO SenderThread:9504 [job_builder.py:build():296] Attempting to build job artifact +2024-08-03 19:20:47,876 INFO SenderThread:9504 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-03 19:20:47,890 INFO SenderThread:9504 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-03 19:20:47,898 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 9 +2024-08-03 19:20:47,899 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:47,899 DEBUG SenderThread:9504 [sender.py:send():382] send: artifact +2024-08-03 19:20:47,899 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-03 19:20:48,494 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:48,494 INFO Thread-12 :9504 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:48,496 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:20:49,057 INFO wandb-upload_1:9504 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmp1gtfugn3 +2024-08-03 19:20:49,437 INFO wandb-upload_0:9504 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp8rydwr53 +2024-08-03 19:20:50,875 INFO SenderThread:9504 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk4ODAyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'versionIndex': 1}}} +2024-08-03 19:20:50,876 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:50,876 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-03 19:20:50,876 INFO SenderThread:9504 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-03 19:20:51,495 INFO SenderThread:9504 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240803_191815-jdwps0z3/files +2024-08-03 19:20:51,495 INFO SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt requirements.txt +2024-08-03 19:20:51,495 INFO SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/config.yaml config.yaml +2024-08-03 19:20:51,497 INFO SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-metadata.json wandb-metadata.json +2024-08-03 19:20:51,497 INFO SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json wandb-summary.json +2024-08-03 19:20:51,498 INFO SenderThread:9504 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_191815-jdwps0z3/files/output.log output.log +2024-08-03 19:20:51,500 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 10 +2024-08-03 19:20:51,500 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:20:51,502 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:51,502 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-03 19:20:51,502 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:51,502 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-03 19:20:51,502 INFO SenderThread:9504 [file_pusher.py:finish():172] shutting down file pusher +2024-08-03 19:20:51,911 INFO wandb-upload_0:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/config.yaml +2024-08-03 19:20:51,994 INFO wandb-upload_1:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/requirements.txt +2024-08-03 19:20:52,073 INFO wandb-upload_2:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/wandb-summary.json +2024-08-03 19:20:52,122 INFO wandb-upload_3:9504 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240803_191815-jdwps0z3/files/output.log +2024-08-03 19:20:52,322 INFO Thread-11 (_thread_body):9504 [sender.py:transition_state():617] send defer: 11 +2024-08-03 19:20:52,322 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:52,322 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-03 19:20:52,323 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:52,323 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-03 19:20:52,323 INFO SenderThread:9504 [file_pusher.py:join():178] waiting for file pusher +2024-08-03 19:20:52,323 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 12 +2024-08-03 19:20:52,323 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:52,323 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-03 19:20:52,323 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:52,323 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-03 19:20:52,323 INFO SenderThread:9504 [file_stream.py:finish():595] file stream finish called +2024-08-03 19:20:52,498 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 19:20:52,502 INFO SenderThread:9504 [file_stream.py:finish():599] file stream finish is done +2024-08-03 19:20:52,502 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 13 +2024-08-03 19:20:52,502 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 19:20:52,502 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:52,502 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-03 19:20:52,503 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:52,503 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-03 19:20:52,503 INFO SenderThread:9504 [sender.py:transition_state():617] send defer: 14 +2024-08-03 19:20:52,503 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: defer +2024-08-03 19:20:52,503 DEBUG SenderThread:9504 [sender.py:send():382] send: final +2024-08-03 19:20:52,503 INFO HandlerThread:9504 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-03 19:20:52,503 DEBUG SenderThread:9504 [sender.py:send():382] send: footer +2024-08-03 19:20:52,503 DEBUG SenderThread:9504 [sender.py:send_request():409] send_request: defer +2024-08-03 19:20:52,503 INFO SenderThread:9504 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-03 19:20:56,504 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:21:01,505 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:21:06,505 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:21:11,506 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:21:16,506 DEBUG HandlerThread:9504 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:21:21,065 WARNING StreamThr :9504 [internal.py:is_dead():414] Internal process exiting, parent pid 9433 disappeared +2024-08-03 19:21:21,065 ERROR StreamThr :9504 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-03 19:21:21,507 INFO SenderThread:9504 [sender.py:finish():1572] shutting down sender +2024-08-03 19:21:21,507 INFO SenderThread:9504 [file_pusher.py:finish():172] shutting down file pusher +2024-08-03 19:21:21,507 INFO SenderThread:9504 [file_pusher.py:join():178] waiting for file pusher +2024-08-03 19:21:21,507 INFO HandlerThread:9504 [handler.py:finish():869] shutting down handler +2024-08-03 19:21:21,507 INFO WriterThread:9504 [datastore.py:close():296] close: /project/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb diff --git a/wandb/run-20240803_191815-jdwps0z3/logs/debug.log b/wandb/run-20240803_191815-jdwps0z3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e933ad8622fe94cb9084689c68e61e92300a2da9 --- /dev/null +++ b/wandb/run-20240803_191815-jdwps0z3/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-03 19:18:15,865 INFO MainThread:9433 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_setup.py:_flush():76] Configure stats pid to 9433 +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240803_191815-jdwps0z3/logs/debug.log +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240803_191815-jdwps0z3/logs/debug-internal.log +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_init.py:init():566] calling init triggers +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample2_train_2024-08-03-19:18:05', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample2', 'save': '/work/llm_recipes/models/tiny-mistral-sample2', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 1600, 'micro_batch_size': 40, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40} +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_init.py:init():616] starting backend +2024-08-03 19:18:15,866 INFO MainThread:9433 [wandb_init.py:init():620] setting up manager +2024-08-03 19:18:15,871 INFO MainThread:9433 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-03 19:18:15,872 INFO MainThread:9433 [wandb_init.py:init():628] backend started and connected +2024-08-03 19:18:15,877 INFO MainThread:9433 [wandb_init.py:init():720] updated telemetry +2024-08-03 19:18:15,887 INFO MainThread:9433 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-03 19:18:16,387 INFO MainThread:9433 [wandb_run.py:_on_init():2262] communicating current version +2024-08-03 19:18:16,470 INFO MainThread:9433 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-03 19:18:16,470 INFO MainThread:9433 [wandb_init.py:init():804] starting run threads in backend +2024-08-03 19:18:16,525 INFO MainThread:9433 [wandb_run.py:_console_start():2241] atexit reg +2024-08-03 19:18:16,525 INFO MainThread:9433 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-03 19:18:16,525 INFO MainThread:9433 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-03 19:18:16,525 INFO MainThread:9433 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-03 19:18:16,526 INFO MainThread:9433 [wandb_init.py:init():847] run started, returning control to user process +2024-08-03 19:18:17,990 INFO MainThread:9433 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'} +2024-08-03 19:18:17,990 INFO MainThread:9433 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb b/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8e3b3c826c1d73a46f3a5261cb62fd09685d7979 Binary files /dev/null and b/wandb/run-20240803_191815-jdwps0z3/run-jdwps0z3.wandb differ diff --git a/wandb/run-20240803_192355-n3hnzq4n/files/config.yaml b/wandb/run-20240803_192355-n3hnzq4n/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e38634bdd5bdc8d7a438c9d06baef3e65275f1db --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-mistral-sample2_train_2024-08-03-19:23:42 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample2 +save: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample2 +base_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1600 +micro_batch_size: + desc: null + value: 40 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-mistral-sample2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32768 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722680635.371313 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 256 +model_type: + desc: null + value: mistral +max_position_embeddings: + desc: null + value: 512 +num_attention_heads: + desc: null + value: 4 +num_hidden_layers: + desc: null + value: 4 +model_architecture: + desc: null + value: MistralForCausalLM diff --git a/wandb/run-20240803_192355-n3hnzq4n/files/output.log b/wandb/run-20240803_192355-n3hnzq4n/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ae51ad48dd314a3c25f0fa1d0febb78a21f02577 --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/files/output.log @@ -0,0 +1,8755 @@ +Created Hugging Face repository with ID koichi12/tiny-mistral-sample2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample2, skipping model loading +--> Model /share/pretrained_lm/custom/tiny-mistral +--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 32000000 + validation: 1616000 + test: 16000 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample2, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample2/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): MistralForCausalLM( + (model): MistralModel( + (embed_tokens): Embedding(32768, 256) + (layers): ModuleList( + (0-3): 4 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): MistralDecoderLayer( + (self_attn): MistralFlashAttention2( + (q_proj): Linear(in_features=256, out_features=512, bias=False) + (k_proj): Linear(in_features=256, out_features=256, bias=False) + (v_proj): Linear(in_features=256, out_features=256, bias=False) + (o_proj): Linear(in_features=512, out_features=256, bias=False) + (rotary_emb): MistralRotaryEmbedding() + ) + (mlp): MistralMLP( + (gate_proj): Linear(in_features=256, out_features=512, bias=False) + (up_proj): Linear(in_features=256, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=256, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): MistralRMSNorm() + (post_attention_layernorm): MistralRMSNorm() + ) + ) + ) + ) + (norm): MistralRMSNorm() + ) + (lm_head): Linear(in_features=256, out_features=32768, bias=False) + ) +) +model config: MistralConfig { + "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 256, + "initializer_range": 0.02, + "intermediate_size": 512, + "label_smoothing": 0.0, + "max_position_embeddings": 512, + "model_type": "mistral", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32768 +} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 11.777307722610205, Tokens per sec: 166366.9078409641, Loss: 10.45809555053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 19.94032335103109, Tokens per sec: 281678.1233363898, Loss: 10.456090927124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 19.950385725832977, Tokens per sec: 281820.2650058405, Loss: 10.458115577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 19.96119078469896, Tokens per sec: 281972.8978719345, Loss: 10.456514358520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 19.827768312242107, Tokens per sec: 280088.16456089704, Loss: 10.458465576171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 19.853387077355965, Tokens per sec: 280450.0566702892, Loss: 10.457174301147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 19.980207721225845, Tokens per sec: 282241.53218132874, Loss: 10.457324981689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 19.982923288895616, Tokens per sec: 282279.8924421704, Loss: 10.457308769226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 19.980131521641077, Tokens per sec: 282240.45578173077, Loss: 10.457944869995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 19.814233490174193, Tokens per sec: 279896.97090707975, Loss: 10.456637382507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 19.989646878956354, Tokens per sec: 282374.8702515595, Loss: 10.456350326538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 19.986021881086085, Tokens per sec: 282323.66332882206, Loss: 10.455950736999512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 19.983754032363404, Tokens per sec: 282291.62757087726, Loss: 10.458043098449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 19.984043803369396, Tokens per sec: 282295.72089232085, Loss: 10.457956314086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 19.98364107061637, Tokens per sec: 282290.0318669183, Loss: 10.459122657775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 19.376722207995293, Tokens per sec: 273716.6620558555, Loss: 10.456958770751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 19.9775261422607, Tokens per sec: 282203.65204683016, Loss: 10.4567289352417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 19.448456974283737, Tokens per sec: 274729.9913780733, Loss: 10.4560546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 19.955115209954453, Tokens per sec: 281887.0739631595, Loss: 10.458041191101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 19.90894216961987, Tokens per sec: 281234.8310119685, Loss: 10.455198287963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 19.992897356142006, Tokens per sec: 282420.7866741513, Loss: 10.457011222839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 19.9895971795937, Tokens per sec: 282374.168195582, Loss: 10.457305908203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 19.96826737155665, Tokens per sec: 282072.8621338279, Loss: 10.456291198730469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 19.936968405298416, Tokens per sec: 281630.73118525767, Loss: 10.456795692443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 19.957390663026327, Tokens per sec: 281919.2171405666, Loss: 10.45568561553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 19.694294947885822, Tokens per sec: 278202.71234804066, Loss: 10.45527172088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 19.87855012965774, Tokens per sec: 280805.5113550004, Loss: 10.456526756286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 19.931200125738567, Tokens per sec: 281549.24814545456, Loss: 10.458826065063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 19.91220415227752, Tokens per sec: 281280.9099615014, Loss: 10.454756736755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 19.882393704632097, Tokens per sec: 280859.80591013975, Loss: 10.455425262451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 19.887339444842194, Tokens per sec: 280929.66971306637, Loss: 10.455729484558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 19.3213339301647, Tokens per sec: 272934.24414419464, Loss: 10.456138610839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 19.896387029425494, Tokens per sec: 281057.4763991109, Loss: 10.454835891723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 19.899614196730557, Tokens per sec: 281103.06354502524, Loss: 10.455574989318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 19.694520211849074, Tokens per sec: 278205.8944393893, Loss: 10.455390930175781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 19.663332613164023, Tokens per sec: 277765.336675389, Loss: 10.45545482635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 19.88679943592819, Tokens per sec: 280922.0415169331, Loss: 10.455411911010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 19.855189388755846, Tokens per sec: 280475.51622196496, Loss: 10.45567798614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 19.862186997532525, Tokens per sec: 280574.36483506707, Loss: 10.454792976379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 19.89434665250417, Tokens per sec: 281028.65392055927, Loss: 10.454419136047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 19.57529545140406, Tokens per sec: 276521.7188026063, Loss: 10.453974723815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 19.785531495420823, Tokens per sec: 279491.5249232902, Loss: 10.454325675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 19.78087876564781, Tokens per sec: 279425.80020219204, Loss: 10.453246116638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 19.892216188122852, Tokens per sec: 280998.55886150734, Loss: 10.45288372039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 19.89271344272225, Tokens per sec: 281005.5831078003, Loss: 10.452845573425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 19.309992578999804, Tokens per sec: 272774.03558308014, Loss: 10.452916145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 19.866288599702933, Tokens per sec: 280632.3042968147, Loss: 10.452441215515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 19.572328059637425, Tokens per sec: 276479.80126048246, Loss: 10.452781677246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 19.731179817652624, Tokens per sec: 278723.7500821095, Loss: 10.451306343078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 19.596463670732206, Tokens per sec: 276820.74225321726, Loss: 10.451889038085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 19.714931423561094, Tokens per sec: 278494.22435805964, Loss: 10.451807975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 19.712240405087982, Tokens per sec: 278456.2108805435, Loss: 10.45233154296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 19.816910006105765, Tokens per sec: 279934.7795208827, Loss: 10.451435089111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 19.835272203127563, Tokens per sec: 280194.1649433947, Loss: 10.451841354370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 19.816366894758925, Tokens per sec: 279927.1074996096, Loss: 10.450197219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 19.82918677461307, Tokens per sec: 280108.2018397135, Loss: 10.450492858886719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 19.856034257481195, Tokens per sec: 280487.4508848504, Loss: 10.450324058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 19.754075067924934, Tokens per sec: 279047.1696684667, Loss: 10.451021194458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 19.88608828776686, Tokens per sec: 280911.99579821684, Loss: 10.449101448059082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 19.672630318759047, Tokens per sec: 277896.67658483965, Loss: 10.449374198913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 18.942735627534134, Tokens per sec: 267586.14333830646, Loss: 10.45008373260498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 19.80241016861832, Tokens per sec: 279729.95400525554, Loss: 10.449317932128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 19.886104295137216, Tokens per sec: 280912.2219192261, Loss: 10.448343276977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 19.88298222965904, Tokens per sec: 280868.11944759893, Loss: 10.448972702026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 19.618891905680496, Tokens per sec: 277137.5647549776, Loss: 10.449479103088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 19.72664239382679, Tokens per sec: 278659.6541792726, Loss: 10.447696685791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 19.884942541486502, Tokens per sec: 280895.81092215487, Loss: 10.446887016296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 19.816848678945217, Tokens per sec: 279933.91320998146, Loss: 10.448254585266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 19.807630004858183, Tokens per sec: 279803.6897040346, Loss: 10.447774887084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 19.814638511822807, Tokens per sec: 279902.6922655494, Loss: 10.446672439575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 19.81802494388816, Tokens per sec: 279950.52919437864, Loss: 10.447793960571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 19.814394592778573, Tokens per sec: 279899.24665148306, Loss: 10.448819160461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 19.816388960337363, Tokens per sec: 279927.41919920524, Loss: 10.447441101074219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 19.80665722201167, Tokens per sec: 279789.94811911666, Loss: 10.446059226989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 19.815103041532023, Tokens per sec: 279909.25423821266, Loss: 10.446627616882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 19.207967093168175, Tokens per sec: 271332.8178618013, Loss: 10.446633338928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 19.710003171017696, Tokens per sec: 278424.60758689133, Loss: 10.444908142089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 19.812766565533416, Tokens per sec: 279876.24904752825, Loss: 10.444125175476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 19.637358278270998, Tokens per sec: 277398.4217674018, Loss: 10.44405746459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 19.43747079108201, Tokens per sec: 274574.79993947846, Loss: 10.442763328552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 19.734822279121783, Tokens per sec: 278775.20369662193, Loss: 10.44226360321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 19.810814567937655, Tokens per sec: 279848.6750202745, Loss: 10.44242000579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 19.815701206463725, Tokens per sec: 279917.70394950575, Loss: 10.441398620605469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 19.6965199240877, Tokens per sec: 278234.1424863579, Loss: 10.441900253295898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 19.81399318236536, Tokens per sec: 279893.57630552666, Loss: 10.438960075378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 19.81118904021332, Tokens per sec: 279853.9648365926, Loss: 10.43940544128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 19.847480413990716, Tokens per sec: 280366.61881310836, Loss: 10.438587188720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 19.862798179852152, Tokens per sec: 280582.99843071034, Loss: 10.4376220703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 19.854281933822204, Tokens per sec: 280462.69746279944, Loss: 10.436944007873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 19.852458529682668, Tokens per sec: 280436.93995390314, Loss: 10.435389518737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 18.9689493974754, Tokens per sec: 267956.44051918166, Loss: 10.435187339782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 19.856889119054934, Tokens per sec: 280499.52670727135, Loss: 10.433349609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 19.855418280555366, Tokens per sec: 280478.7495603317, Loss: 10.432583808898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 19.856477740558297, Tokens per sec: 280493.71555161086, Loss: 10.432395935058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 19.71883270748371, Tokens per sec: 278549.334113031, Loss: 10.431503295898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 19.850347449904604, Tokens per sec: 280407.1187228413, Loss: 10.431466102600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 19.848294455705982, Tokens per sec: 280378.1180119246, Loss: 10.43079948425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 19.85562554698582, Tokens per sec: 280481.67741752503, Loss: 10.4303560256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 19.856750355623635, Tokens per sec: 280497.5665272769, Loss: 10.42905330657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 19.693313635202653, Tokens per sec: 278188.8502701729, Loss: 10.42811107635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 19.854274505242007, Tokens per sec: 280462.5925262599, Loss: 10.427715301513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 19.752083849498757, Tokens per sec: 279019.041605568, Loss: 10.427726745605469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 19.848860507277532, Tokens per sec: 280386.1140880955, Loss: 10.426887512207031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 19.678934280104823, Tokens per sec: 277985.72669552255, Loss: 10.426054000854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 19.85104349756776, Tokens per sec: 280416.95113107556, Loss: 10.42399787902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 19.21447551081224, Tokens per sec: 271424.7561335933, Loss: 10.42393684387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 19.747524559218903, Tokens per sec: 278954.6368159783, Loss: 10.42391586303711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 19.860687215482983, Tokens per sec: 280553.178829921, Loss: 10.42404842376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 19.843821813637096, Tokens per sec: 280314.9372198111, Loss: 10.421424865722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 19.84966733201007, Tokens per sec: 280397.5113394099, Loss: 10.420833587646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 19.594031987791105, Tokens per sec: 276786.3921639364, Loss: 10.42106819152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 19.85599068920883, Tokens per sec: 280486.8354369973, Loss: 10.41929817199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 19.68218064350827, Tokens per sec: 278031.58500659675, Loss: 10.418926239013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 19.851107016879297, Tokens per sec: 280417.8484084243, Loss: 10.417434692382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 19.675849777172903, Tokens per sec: 277942.1548345254, Loss: 10.417305946350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 19.722414158290658, Tokens per sec: 278599.9258875154, Loss: 10.416015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 19.84479400780603, Tokens per sec: 280328.67048903654, Loss: 10.415653228759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 19.73811310453949, Tokens per sec: 278821.69008059724, Loss: 10.413409233093262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 19.84161829621594, Tokens per sec: 280283.810209431, Loss: 10.412717819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 19.256755970712042, Tokens per sec: 272022.01227576926, Loss: 10.412031173706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 19.838285872841187, Tokens per sec: 280236.7362103869, Loss: 10.413021087646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 19.84752483981214, Tokens per sec: 280367.24637474754, Loss: 10.411465644836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 19.85383724632998, Tokens per sec: 280456.41578240355, Loss: 10.410470008850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 19.851315285224047, Tokens per sec: 280420.79041871504, Loss: 10.41059398651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 19.855782261684414, Tokens per sec: 280483.8911781257, Loss: 10.408648490905762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 19.849000570806076, Tokens per sec: 280388.0926333363, Loss: 10.408571243286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 19.869118914058372, Tokens per sec: 280672.2854757584, Loss: 10.407718658447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 19.85076903857811, Tokens per sec: 280413.0741080315, Loss: 10.407212257385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 19.845948804031906, Tokens per sec: 280344.9832051352, Loss: 10.406448364257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 19.82796385123647, Tokens per sec: 280090.926755672, Loss: 10.405963897705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 19.848403214349123, Tokens per sec: 280379.65434260276, Loss: 10.405253410339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 19.71068692371487, Tokens per sec: 278434.2663157482, Loss: 10.404086112976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 19.821812107990226, Tokens per sec: 280004.02688637975, Loss: 10.402256965637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 19.832574891823374, Tokens per sec: 280156.0625729946, Loss: 10.401341438293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 19.226556943149763, Tokens per sec: 271595.41912276065, Loss: 10.401857376098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 19.544899233555217, Tokens per sec: 276092.34012857673, Loss: 10.400236129760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 19.816614694360887, Tokens per sec: 279930.6079306516, Loss: 10.398975372314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 19.8034367587913, Tokens per sec: 279744.45567547774, Loss: 10.397867202758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 19.812159032458627, Tokens per sec: 279867.6670013218, Loss: 10.398712158203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 19.80157419606, Tokens per sec: 279718.14501012326, Loss: 10.398239135742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 19.7886502362856, Tokens per sec: 279535.5804312425, Loss: 10.396282196044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 19.694037875841826, Tokens per sec: 278199.08093396376, Loss: 10.396781921386719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 19.66348184206594, Tokens per sec: 277767.44469120697, Loss: 10.39403247833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 19.706805203621087, Tokens per sec: 278379.4329205175, Loss: 10.39378547668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 19.435176199960523, Tokens per sec: 274542.3864169118, Loss: 10.391814231872559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 19.39684722997779, Tokens per sec: 274000.94924239407, Loss: 10.393048286437988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 19.67901973646296, Tokens per sec: 277986.93385681894, Loss: 10.390438079833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 19.690907404769256, Tokens per sec: 278154.8597244399, Loss: 10.389171600341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 19.308711970385968, Tokens per sec: 272755.94563414983, Loss: 10.389507293701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 19.64040294731028, Tokens per sec: 277441.4309326028, Loss: 10.389298439025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 19.671949816523476, Tokens per sec: 277887.0637721852, Loss: 10.387598037719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 19.787860520878127, Tokens per sec: 279524.4248672112, Loss: 10.387393951416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 19.82811514963732, Tokens per sec: 280093.0640053476, Loss: 10.385275840759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 19.74164827978726, Tokens per sec: 278871.62816394353, Loss: 10.386504173278809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 19.66924214071827, Tokens per sec: 277848.8149922639, Loss: 10.384217262268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 19.80259771657768, Tokens per sec: 279732.603318223, Loss: 10.382808685302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 19.84121084293488, Tokens per sec: 280278.0545015853, Loss: 10.382784843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 19.808408063589926, Tokens per sec: 279814.6806052123, Loss: 10.382719039916992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 19.80826001670582, Tokens per sec: 279812.589286644, Loss: 10.382160186767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 19.67856185180303, Tokens per sec: 277980.4657524937, Loss: 10.380424499511719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 19.798734129415156, Tokens per sec: 279678.02606979385, Loss: 10.378926277160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 19.661056032076825, Tokens per sec: 277733.1775635744, Loss: 10.378963470458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 19.73376942099514, Tokens per sec: 278760.3309638166, Loss: 10.376039505004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 19.484586674354997, Tokens per sec: 275240.36154277035, Loss: 10.377671241760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 19.50280789638209, Tokens per sec: 275497.7555446195, Loss: 10.375601768493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 19.796797117406857, Tokens per sec: 279650.663729787, Loss: 10.375688552856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 19.566092749922145, Tokens per sec: 276391.720926573, Loss: 10.372477531433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 19.766318266249318, Tokens per sec: 279220.11777301534, Loss: 10.372026443481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 19.723067949815295, Tokens per sec: 278609.16138317273, Loss: 10.372231483459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 19.726861098325962, Tokens per sec: 278662.7436112647, Loss: 10.372282028198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 19.787147273719217, Tokens per sec: 279514.3494979376, Loss: 10.370718955993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 19.79576793781767, Tokens per sec: 279636.12548132654, Loss: 10.369728088378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 19.7450423418451, Tokens per sec: 278919.5728744736, Loss: 10.366216659545898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 19.7963481959337, Tokens per sec: 279644.3222399396, Loss: 10.366242408752441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 19.547129173128837, Tokens per sec: 276123.8403797609, Loss: 10.362969398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 19.760599944878678, Tokens per sec: 279139.3404453883, Loss: 10.362947463989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 19.748901955399592, Tokens per sec: 278974.0939914933, Loss: 10.35920524597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 19.68662549793724, Tokens per sec: 278094.3732689541, Loss: 10.356941223144531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 19.448693893827322, Tokens per sec: 274733.33811680187, Loss: 10.355287551879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 19.494586312248682, Tokens per sec: 275381.6169871457, Loss: 10.353496551513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 19.64951648078178, Tokens per sec: 277570.1692163319, Loss: 10.350483894348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 19.797271765129164, Tokens per sec: 279657.36863006925, Loss: 10.347902297973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 19.641860692723384, Tokens per sec: 277462.0231258705, Loss: 10.345388412475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 19.773257917607005, Tokens per sec: 279318.147676374, Loss: 10.343332290649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 19.77502718146978, Tokens per sec: 279343.1403966916, Loss: 10.341137886047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 19.642259322184874, Tokens per sec: 277467.6541879472, Loss: 10.339000701904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 19.79323704761199, Tokens per sec: 279600.37398467545, Loss: 10.335359573364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 19.684315299484556, Tokens per sec: 278061.73927635374, Loss: 10.33210277557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 19.785194567377843, Tokens per sec: 279486.7654589036, Loss: 10.329109191894531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 19.692678346533253, Tokens per sec: 278179.87614688394, Loss: 10.326434135437012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 19.77724306713721, Tokens per sec: 279374.44212161057, Loss: 10.323370933532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 19.79083291063865, Tokens per sec: 279566.4130112763, Loss: 10.320558547973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 19.676512998496392, Tokens per sec: 277951.52353604895, Loss: 10.31836986541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 19.440149121162666, Tokens per sec: 274612.6341800529, Loss: 10.315072059631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 19.665683946981673, Tokens per sec: 277798.55174845643, Loss: 10.311816215515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 19.839205725282305, Tokens per sec: 280249.73009743687, Loss: 10.308269500732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 19.791030721519604, Tokens per sec: 279569.2072988484, Loss: 10.306909561157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 19.843868838995885, Tokens per sec: 280315.60150266043, Loss: 10.302423477172852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 19.760397856337633, Tokens per sec: 279136.48573135043, Loss: 10.302054405212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 19.77823255351791, Tokens per sec: 279388.419661587, Loss: 10.298306465148926 +------------------------------------------------------------------ +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +[rank0]:[2024-08-03 19:33:48,912] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0014889889998812578, 'preprocessing_with_comm': 0.000674175000312971, 'state_converting': 0.02897366300021531, : 0.03153780299999198}) + eval ppl=29548.970703125, eval loss=10.293804168701172 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200, took 0.24s +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 19.466721765042408, Tokens per sec: 274988.0008342624, Loss: 10.294522285461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 19.783834521743362, Tokens per sec: 279467.55337817513, Loss: 10.29236888885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 19.738374359246095, Tokens per sec: 278825.38057920017, Loss: 10.289491653442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 19.650040005157734, Tokens per sec: 277577.56455095834, Loss: 10.287432670593262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 19.781063481684022, Tokens per sec: 279428.4095112546, Loss: 10.283553123474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 19.820160415789253, Tokens per sec: 279980.6949899351, Loss: 10.280791282653809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 19.71190102006409, Tokens per sec: 278451.4167087071, Loss: 10.27712631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 18.88273852607481, Tokens per sec: 266738.62092619814, Loss: 10.275691986083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 19.806308360089773, Tokens per sec: 279785.0200760888, Loss: 10.27358341217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 19.82845895426095, Tokens per sec: 280097.9206086972, Loss: 10.269571304321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 19.61872014203957, Tokens per sec: 277135.1384121755, Loss: 10.265687942504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 19.776945920791434, Tokens per sec: 279370.2446157045, Loss: 10.261597633361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 19.738514329550284, Tokens per sec: 278827.35780754854, Loss: 10.26020622253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 19.779680518455308, Tokens per sec: 279408.8736953077, Loss: 10.258440971374512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 19.776831784715892, Tokens per sec: 279368.63232311537, Loss: 10.254993438720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 19.762107957523646, Tokens per sec: 279160.64271638583, Loss: 10.253373146057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 19.786999059316276, Tokens per sec: 279512.2558129889, Loss: 10.251169204711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 19.633898280234142, Tokens per sec: 277349.54564154305, Loss: 10.24579906463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 19.792197700804092, Tokens per sec: 279585.69211351464, Loss: 10.246060371398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 19.739505161490154, Tokens per sec: 278841.35435496917, Loss: 10.24205493927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 19.762843484666192, Tokens per sec: 279171.0328139549, Loss: 10.240159034729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 19.522268573406496, Tokens per sec: 275772.65815710934, Loss: 10.237457275390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 19.41507883903455, Tokens per sec: 274258.48997200537, Loss: 10.234665870666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 19.768862286049526, Tokens per sec: 279256.0547390534, Loss: 10.22946834564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 19.72885307552741, Tokens per sec: 278690.8823926653, Loss: 10.227843284606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 19.794291432934614, Tokens per sec: 279615.26829073665, Loss: 10.228028297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 19.78301794558355, Tokens per sec: 279456.0183776534, Loss: 10.223095893859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 19.79121999541641, Tokens per sec: 279571.8809925047, Loss: 10.222803115844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 19.767260131038515, Tokens per sec: 279233.42260772584, Loss: 10.219437599182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 19.657103858437345, Tokens per sec: 277677.3489376153, Loss: 10.217411994934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 19.650430261076988, Tokens per sec: 277583.07732790883, Loss: 10.214521408081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 19.77102061028135, Tokens per sec: 279286.54334791243, Loss: 10.213553428649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 19.80557252399717, Tokens per sec: 279774.625614274, Loss: 10.211841583251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 19.7416239120485, Tokens per sec: 278871.28394390247, Loss: 10.206798553466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 19.456090124000305, Tokens per sec: 274837.81767805136, Loss: 10.202709197998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 19.77308179858346, Tokens per sec: 279315.6598091934, Loss: 10.20250415802002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 19.752798982761597, Tokens per sec: 279029.14361805125, Loss: 10.201128959655762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 19.40591382826092, Tokens per sec: 274129.0245170262, Loss: 10.197005271911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 19.78093497471546, Tokens per sec: 279426.59421462665, Loss: 10.195477485656738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 19.75817969015538, Tokens per sec: 279105.1517917515, Loss: 10.195076942443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 19.770057238468134, Tokens per sec: 279272.93470377737, Loss: 10.188849449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 19.758956610162816, Tokens per sec: 279116.12660724594, Loss: 10.18994426727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 19.81825599435134, Tokens per sec: 279953.793026149, Loss: 10.18967056274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 19.758988304244657, Tokens per sec: 279116.5743196193, Loss: 10.182967185974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 19.791712085392355, Tokens per sec: 279578.83228303783, Loss: 10.18227481842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 19.779277604679482, Tokens per sec: 279403.182112767, Loss: 10.179178237915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 19.8130261520231, Tokens per sec: 279879.91598080564, Loss: 10.174702644348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 19.79484620155322, Tokens per sec: 279623.104983283, Loss: 10.174427032470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 19.77415263777444, Tokens per sec: 279330.7865435197, Loss: 10.170499801635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 19.78739070431749, Tokens per sec: 279517.788212189, Loss: 10.168869972229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 19.82496276594783, Tokens per sec: 280048.53325697116, Loss: 10.16802978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 19.818500907509755, Tokens per sec: 279957.2526831279, Loss: 10.165196418762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 19.43270248258773, Tokens per sec: 274507.4425468969, Loss: 10.162671089172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 19.7811836264013, Tokens per sec: 279430.1066822531, Loss: 10.161419868469238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 19.597552981905018, Tokens per sec: 276836.12992379244, Loss: 10.158500671386719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 19.673560151959133, Tokens per sec: 277909.81146064907, Loss: 10.156441688537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 19.811322364944168, Tokens per sec: 279855.8481892002, Loss: 10.155837059020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 19.694018858705306, Tokens per sec: 278198.8122968293, Loss: 10.151531219482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 19.793717687530044, Tokens per sec: 279607.1635310502, Loss: 10.151250839233398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 19.85097506860581, Tokens per sec: 280415.98449973034, Loss: 10.147686004638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 19.84859118702642, Tokens per sec: 280382.3096551595, Loss: 10.145214080810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 19.855801063273404, Tokens per sec: 280484.15677042375, Loss: 10.144444465637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 19.711218981852156, Tokens per sec: 278441.7821987646, Loss: 10.140650749206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 19.687313699718356, Tokens per sec: 278104.09484581964, Loss: 10.13705825805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 19.668551380217572, Tokens per sec: 277839.05727078236, Loss: 10.13670825958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 19.774566440093178, Tokens per sec: 279336.6319382268, Loss: 10.135492324829102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 19.667017910732362, Tokens per sec: 277817.3953950352, Loss: 10.133078575134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 18.990341997433653, Tokens per sec: 268258.6335831279, Loss: 10.132878303527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 19.680170743708207, Tokens per sec: 278003.19304956525, Loss: 10.130179405212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 19.73419951707056, Tokens per sec: 278766.4065250422, Loss: 10.128682136535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 19.713628895144215, Tokens per sec: 278475.8247687652, Loss: 10.123601913452148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 19.68320928094988, Tokens per sec: 278046.11559665017, Loss: 10.12299919128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 19.745691843677545, Tokens per sec: 278928.7477736989, Loss: 10.120275497436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 19.759185989717725, Tokens per sec: 279119.3668356726, Loss: 10.117508888244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 19.785693014216154, Tokens per sec: 279493.80654683016, Loss: 10.117244720458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 19.751170943518552, Tokens per sec: 279006.1458446138, Loss: 10.114633560180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 19.791358275909992, Tokens per sec: 279573.834350494, Loss: 10.112789154052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 19.788355900972512, Tokens per sec: 279531.4226341415, Loss: 10.111066818237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 19.802715808147536, Tokens per sec: 279734.27148634614, Loss: 10.108184814453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 19.78173219201062, Tokens per sec: 279437.85575074307, Loss: 10.106047630310059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 19.802172583730908, Tokens per sec: 279726.5978678429, Loss: 10.108725547790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 19.6525052785583, Tokens per sec: 277612.38914094906, Loss: 10.103455543518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 19.263763609747084, Tokens per sec: 272121.00257686217, Loss: 10.100166320800781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 19.788069802631444, Tokens per sec: 279527.3811929681, Loss: 10.09984016418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 19.70419048502336, Tokens per sec: 278342.4972593101, Loss: 10.093533515930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 19.71795776727996, Tokens per sec: 278536.9746587591, Loss: 10.095112800598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 19.79952980002141, Tokens per sec: 279689.26575729635, Loss: 10.09154224395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 19.7814613163876, Tokens per sec: 279434.02934653655, Loss: 10.090241432189941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 19.78323474964769, Tokens per sec: 279459.08096399385, Loss: 10.08526611328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 19.72424012462533, Tokens per sec: 278625.71959012357, Loss: 10.085308074951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 19.794559902026844, Tokens per sec: 279619.06070015463, Loss: 10.0850191116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 19.774857727201542, Tokens per sec: 279340.74667621736, Loss: 10.085107803344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 19.80137256418559, Tokens per sec: 279715.2967469839, Loss: 10.08005428314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 19.753575224476663, Tokens per sec: 279040.1088519498, Loss: 10.079572677612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 19.674202234907405, Tokens per sec: 277918.88156030147, Loss: 10.0752592086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 19.722571187214033, Tokens per sec: 278602.14408687304, Loss: 10.07448673248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 19.155593172624684, Tokens per sec: 270592.9809298341, Loss: 10.073501586914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 19.693171535327604, Tokens per sec: 278186.8429593873, Loss: 10.072277069091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 19.81770739516471, Tokens per sec: 279946.043483344, Loss: 10.06963062286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 19.76394814523334, Tokens per sec: 279186.6373109332, Loss: 10.06611156463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 19.77351562870907, Tokens per sec: 279321.78811782104, Loss: 10.066093444824219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 19.769573946542728, Tokens per sec: 279266.10769499844, Loss: 10.064348220825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 19.620358298736296, Tokens per sec: 277158.2791053298, Loss: 10.059760093688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 19.7551684990155, Tokens per sec: 279062.61553723057, Loss: 10.061529159545898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 19.650353240025364, Tokens per sec: 277581.98932422424, Loss: 10.057979583740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 19.634455804809328, Tokens per sec: 277357.42126488616, Loss: 10.056215286254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 19.639279089576696, Tokens per sec: 277425.5552553773, Loss: 10.05421257019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 19.72016412687823, Tokens per sec: 278568.1418178922, Loss: 10.052546501159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 19.676633655627732, Tokens per sec: 277953.22794543713, Loss: 10.050596237182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 19.80146702969182, Tokens per sec: 279716.6311720103, Loss: 10.047712326049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 19.806010454484813, Tokens per sec: 279780.811844845, Loss: 10.04840087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 19.200080814466702, Tokens per sec: 271221.4158476196, Loss: 10.044208526611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 19.753652244254866, Tokens per sec: 279041.19683764596, Loss: 10.042987823486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 19.805531127878062, Tokens per sec: 279774.0408503793, Loss: 10.040532112121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 19.589964762738155, Tokens per sec: 276728.9383152734, Loss: 10.038477897644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 19.76502670864441, Tokens per sec: 279201.8731580246, Loss: 10.036800384521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 19.593057516090216, Tokens per sec: 276772.626722167, Loss: 10.038108825683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 19.787468223859417, Tokens per sec: 279518.8832575755, Loss: 10.033804893493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 19.790742678644634, Tokens per sec: 279565.1383890803, Loss: 10.033618927001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 19.816730033518507, Tokens per sec: 279932.2372180454, Loss: 10.032071113586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 19.808290167472663, Tokens per sec: 279813.0151980634, Loss: 10.031853675842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 19.703244117788753, Tokens per sec: 278329.1288228039, Loss: 10.030872344970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 19.797035058965307, Tokens per sec: 279654.0249055547, Loss: 10.027664184570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 19.811506089615477, Tokens per sec: 279858.4434941867, Loss: 10.024462699890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 19.800997497993233, Tokens per sec: 279709.9985409654, Loss: 10.022279739379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 19.195368155145353, Tokens per sec: 271154.84455836855, Loss: 10.022287368774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 19.631155344164007, Tokens per sec: 277310.79877314647, Loss: 10.020240783691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 19.78482107040972, Tokens per sec: 279481.4894198344, Loss: 10.01788330078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 19.80179447749887, Tokens per sec: 279721.2567180537, Loss: 10.015453338623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 19.685649537891667, Tokens per sec: 278080.58680274454, Loss: 10.0142822265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 19.821650330280356, Tokens per sec: 280001.7416053985, Loss: 10.014289855957031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 19.713109714982323, Tokens per sec: 278468.4908007496, Loss: 10.013017654418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 19.739070549242747, Tokens per sec: 278835.21499804535, Loss: 10.011130332946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 19.762429010011534, Tokens per sec: 279165.1779217929, Loss: 10.009407043457031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 19.729007045188172, Tokens per sec: 278693.0573767079, Loss: 10.009663581848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 19.80398356245355, Tokens per sec: 279752.1798546049, Loss: 10.002917289733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 19.696635345068874, Tokens per sec: 278235.7729295959, Loss: 10.004566192626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 19.816043702179886, Tokens per sec: 279922.5420631552, Loss: 10.00253677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 19.745099206272272, Tokens per sec: 278920.37614455336, Loss: 10.003435134887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 19.7354561875684, Tokens per sec: 278784.1583228065, Loss: 9.99942684173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 19.584041019599237, Tokens per sec: 276645.259188254, Loss: 9.997052192687988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 19.153506714960827, Tokens per sec: 270563.5075121352, Loss: 9.992971420288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 19.608790787131493, Tokens per sec: 276994.8757891873, Loss: 9.994800567626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 19.67729193204011, Tokens per sec: 277962.5267948695, Loss: 9.992644309997559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 19.72749360462298, Tokens per sec: 278671.6784306056, Loss: 9.993345260620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 19.75167782929865, Tokens per sec: 279013.30614150414, Loss: 9.99324893951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 19.72185843509817, Tokens per sec: 278592.0757106052, Loss: 9.989104270935059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 19.568958925075673, Tokens per sec: 276432.20867715694, Loss: 9.987677574157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 19.605920196719957, Tokens per sec: 276954.32566842175, Loss: 9.985426902770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 19.69496152279843, Tokens per sec: 278212.1284225517, Loss: 9.985769271850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 19.58389051881968, Tokens per sec: 276643.1332058214, Loss: 9.982613563537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 19.67330005064779, Tokens per sec: 277906.1372549721, Loss: 9.978691101074219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 19.702970971195647, Tokens per sec: 278325.2703387469, Loss: 9.979392051696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 19.74632991995363, Tokens per sec: 278937.76127487584, Loss: 9.980032920837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 19.76758533886393, Tokens per sec: 279238.0165116633, Loss: 9.97585678100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 19.089863318768387, Tokens per sec: 269664.47733661334, Loss: 9.973769187927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 19.779470451285622, Tokens per sec: 279405.90627471526, Loss: 9.97408676147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 19.81163129236973, Tokens per sec: 279860.2121152985, Loss: 9.971256256103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 19.81360751693611, Tokens per sec: 279888.1283740948, Loss: 9.969343185424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 19.647719450264145, Tokens per sec: 277544.78426269424, Loss: 9.967988967895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 19.69507434868337, Tokens per sec: 278213.72220731503, Loss: 9.965332984924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 19.71348226830151, Tokens per sec: 278473.75350978127, Loss: 9.961339950561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 19.668604928750085, Tokens per sec: 277839.8137003487, Loss: 9.963549613952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 19.684349335258336, Tokens per sec: 278062.2200675985, Loss: 9.962008476257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 19.61869534817673, Tokens per sec: 277134.78817268176, Loss: 9.957426071166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 19.71253628566103, Tokens per sec: 278460.39050607313, Loss: 9.956300735473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 19.735085920084668, Tokens per sec: 278778.9279036146, Loss: 9.95508098602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 19.75702414197677, Tokens per sec: 279088.8284535265, Loss: 9.952073097229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 19.7632054643187, Tokens per sec: 279176.14615877933, Loss: 9.949198722839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 19.73437430739505, Tokens per sec: 278768.87562294566, Loss: 9.948773384094238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 19.127791526292985, Tokens per sec: 270200.25331822445, Loss: 9.946267127990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 19.494038291770657, Tokens per sec: 275373.8756192108, Loss: 9.944438934326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 19.76243665904515, Tokens per sec: 279165.2859724698, Loss: 9.941377639770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 19.77338323626948, Tokens per sec: 279319.91793481185, Loss: 9.938472747802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 19.629249277780943, Tokens per sec: 277283.8735727731, Loss: 9.936009407043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 19.80066503616127, Tokens per sec: 279705.3021665255, Loss: 9.932795524597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 19.826515168023885, Tokens per sec: 280070.46257555584, Loss: 9.929275512695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 19.766829903967444, Tokens per sec: 279227.34519604826, Loss: 9.92542552947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 19.789948509456522, Tokens per sec: 279553.91991069453, Loss: 9.922094345092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 19.776503815753824, Tokens per sec: 279363.9994152071, Loss: 9.924371719360352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 19.73841333616556, Tokens per sec: 278825.93116934533, Loss: 9.917964935302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 19.807415562425884, Tokens per sec: 279800.66047823767, Loss: 9.911856651306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 19.840096478852423, Tokens per sec: 280262.3129322068, Loss: 9.911105155944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 19.841079358167235, Tokens per sec: 280276.1971404008, Loss: 9.909300804138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 19.156962474360032, Tokens per sec: 270612.32376276137, Loss: 9.903512001037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 19.700477732034752, Tokens per sec: 278290.050702861, Loss: 9.90162181854248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 19.504879364744493, Tokens per sec: 275527.0172226074, Loss: 9.897726058959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 19.68887602798054, Tokens per sec: 278126.16438226507, Loss: 9.894370079040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 19.372490834428106, Tokens per sec: 273656.8894360961, Loss: 9.89285945892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 19.46773960792474, Tokens per sec: 275002.3789397675, Loss: 9.891037940979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 19.515290882406344, Tokens per sec: 275674.0909036328, Loss: 9.885353088378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 19.730769443827988, Tokens per sec: 278717.9531185018, Loss: 9.880919456481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 19.756110181455913, Tokens per sec: 279075.91779607185, Loss: 9.878013610839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 19.645940257933123, Tokens per sec: 277519.6512922788, Loss: 9.873779296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 19.768119713276402, Tokens per sec: 279245.5651145126, Loss: 9.870784759521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 19.756408891885926, Tokens per sec: 279080.13739631936, Loss: 9.868661880493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 19.767299807180812, Tokens per sec: 279233.9830751318, Loss: 9.863361358642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 19.443557175087047, Tokens per sec: 274660.7765404725, Loss: 9.85996150970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 19.635015152521536, Tokens per sec: 277365.3226419648, Loss: 9.854421615600586 +------------------------------------------------------------------ +[rank0]:[2024-08-03 19:43:39,973] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.001012725999316899, 'preprocessing_with_comm': 0.0004856240002482082, 'state_converting': 0.034108046000255854, : 0.03591702100038674}) +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 19.07811402723393, Tokens per sec: 269498.50618701376, Loss: 9.851438522338867 +------------------------------------------------------------------ + eval ppl=18913.84765625, eval loss=9.847649574279785 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400, took 0.16s +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 19.612896505044116, Tokens per sec: 277052.8733901398, Loss: 9.844963073730469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 19.737647008293226, Tokens per sec: 278815.105978944, Loss: 9.845855712890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 19.78275143290001, Tokens per sec: 279452.25360457407, Loss: 9.841656684875488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 19.781546062709985, Tokens per sec: 279435.22647782817, Loss: 9.838088989257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 19.67239184157199, Tokens per sec: 277893.30784275226, Loss: 9.832378387451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 19.79339600870287, Tokens per sec: 279602.61947793927, Loss: 9.8311128616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 19.736891520123272, Tokens per sec: 278804.4339107849, Loss: 9.824028015136719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 19.7762615750751, Tokens per sec: 279360.5775098258, Loss: 9.822295188903809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 19.740278957056812, Tokens per sec: 278852.28503443836, Loss: 9.81959342956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 19.487408051846952, Tokens per sec: 275280.2164790805, Loss: 9.815252304077148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 19.719355234096305, Tokens per sec: 278556.71535319637, Loss: 9.810564994812012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 19.705611556285596, Tokens per sec: 278362.57139147073, Loss: 9.80787181854248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 19.771162288547455, Tokens per sec: 279288.5447030264, Loss: 9.805795669555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 19.831223025545537, Tokens per sec: 280136.9660343157, Loss: 9.800299644470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 18.980155819956266, Tokens per sec: 268114.74307015626, Loss: 9.797642707824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 19.53676256306841, Tokens per sec: 275977.4010660258, Loss: 9.79491901397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 19.67625918890947, Tokens per sec: 277947.93820762314, Loss: 9.791077613830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 19.44264915727321, Tokens per sec: 274647.94983002974, Loss: 9.787690162658691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 19.49749671774196, Tokens per sec: 275422.7295379836, Loss: 9.783546447753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 19.55877228372295, Tokens per sec: 276288.3116114562, Loss: 9.781389236450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 19.599172704527515, Tokens per sec: 276859.01021618285, Loss: 9.777076721191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 19.694946903128884, Tokens per sec: 278211.9219042816, Loss: 9.773795127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 19.82047230303636, Tokens per sec: 279985.1007266381, Loss: 9.76887035369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 19.828192386398843, Tokens per sec: 280094.1550561624, Loss: 9.7687406539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 19.789333134683698, Tokens per sec: 279545.2270922228, Loss: 9.761725425720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 19.713108170058074, Tokens per sec: 278468.46897706325, Loss: 9.760658264160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 19.638988556988096, Tokens per sec: 277421.45117577515, Loss: 9.757981300354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 19.557704763589513, Tokens per sec: 276273.2317623226, Loss: 9.752667427062988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 19.653682615515038, Tokens per sec: 277629.0202686731, Loss: 9.751219749450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 19.52262758569022, Tokens per sec: 275777.72958471626, Loss: 9.746847152709961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 19.581591869118707, Tokens per sec: 276610.6623515338, Loss: 9.744874000549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 19.849367205265217, Tokens per sec: 280393.2717322197, Loss: 9.741876602172852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 19.763299176135508, Tokens per sec: 279177.46993714676, Loss: 9.737592697143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 19.83069139973978, Tokens per sec: 280129.4562584386, Loss: 9.732332229614258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 19.725007367017575, Tokens per sec: 278636.5576990844, Loss: 9.731011390686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 19.76821733207598, Tokens per sec: 279246.9440831373, Loss: 9.727757453918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 19.53729629600818, Tokens per sec: 275984.94060739584, Loss: 9.726058006286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 19.778653387038492, Tokens per sec: 279394.36437944474, Loss: 9.722891807556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 19.682676620281818, Tokens per sec: 278038.5912022502, Loss: 9.7166748046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 19.617228594355073, Tokens per sec: 277114.0687261308, Loss: 9.71537971496582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 19.778121465725437, Tokens per sec: 279386.8504292151, Loss: 9.712447166442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 19.48759325557736, Tokens per sec: 275282.8326773385, Loss: 9.707635879516602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 19.674836451950824, Tokens per sec: 277927.84054574184, Loss: 9.707441329956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 19.692869361325656, Tokens per sec: 278182.57443252887, Loss: 9.700421333312988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 19.090066080552205, Tokens per sec: 269667.3415609163, Loss: 9.700647354125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 19.715234903809403, Tokens per sec: 278498.5113370273, Loss: 9.6978120803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 19.691911081073357, Tokens per sec: 278169.03771206824, Loss: 9.69501781463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 19.749208569988728, Tokens per sec: 278978.42524633475, Loss: 9.690910339355469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 19.538109787373344, Tokens per sec: 275996.4320319357, Loss: 9.685264587402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 19.725237720114638, Tokens per sec: 278639.8116798219, Loss: 9.685625076293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 19.812576987383693, Tokens per sec: 279873.57105597825, Loss: 9.681221961975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 19.777603965579612, Tokens per sec: 279379.5401932005, Loss: 9.678875923156738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 19.752916595293055, Tokens per sec: 279030.8050192512, Loss: 9.677275657653809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 19.543141229466812, Tokens per sec: 276067.50646446197, Loss: 9.6736478805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 19.666166146467404, Tokens per sec: 277805.3633253714, Loss: 9.671832084655762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 19.763476814920672, Tokens per sec: 279179.9792725651, Loss: 9.668463706970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 19.69620143677996, Tokens per sec: 278229.6435168291, Loss: 9.66711711883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 19.820400312063168, Tokens per sec: 279984.08377812285, Loss: 9.663178443908691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 19.804479581402166, Tokens per sec: 279759.18664602574, Loss: 9.658612251281738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 19.191931217156803, Tokens per sec: 271106.2941800424, Loss: 9.654365539550781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 19.631055652015494, Tokens per sec: 277309.39051627874, Loss: 9.653178215026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 19.703771230677887, Tokens per sec: 278336.57484896824, Loss: 9.650996208190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 19.604965233244275, Tokens per sec: 276940.8358009332, Loss: 9.64952278137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 19.789143047678536, Tokens per sec: 279542.5419125524, Loss: 9.643295288085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 19.786414962832737, Tokens per sec: 279504.00483338174, Loss: 9.640857696533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 19.72944579653453, Tokens per sec: 278699.2552027752, Loss: 9.637958526611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 19.766091639273174, Tokens per sec: 279216.9164276704, Loss: 9.63703441619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 19.72831693019978, Tokens per sec: 278683.30877376936, Loss: 9.632867813110352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 19.78283818394948, Tokens per sec: 279453.4790547527, Loss: 9.62879753112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 19.804021739199513, Tokens per sec: 279752.7191414544, Loss: 9.629509925842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 19.783862900714478, Tokens per sec: 279467.95426110894, Loss: 9.625452995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 19.784602277968837, Tokens per sec: 279478.39874557283, Loss: 9.623055458068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 19.452398531180958, Tokens per sec: 274785.6700313372, Loss: 9.621102333068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 19.220186336445295, Tokens per sec: 271505.427576012, Loss: 9.616849899291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 19.77290228475214, Tokens per sec: 279313.12398676825, Loss: 9.613825798034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 19.7616039370505, Tokens per sec: 279153.5228949817, Loss: 9.61334228515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 19.809563565735886, Tokens per sec: 279831.0032931775, Loss: 9.609061241149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 19.79617663600524, Tokens per sec: 279641.8987747912, Loss: 9.608297348022461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 19.661269905061417, Tokens per sec: 277736.1987453211, Loss: 9.603315353393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 19.755888087388403, Tokens per sec: 279072.7804828479, Loss: 9.602185249328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 19.72216246335734, Tokens per sec: 278596.37043080485, Loss: 9.599303245544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 19.634676948496317, Tokens per sec: 277360.54515298177, Loss: 9.599993705749512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 19.76461664679244, Tokens per sec: 279196.08060136036, Loss: 9.59521198272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 19.76553846776311, Tokens per sec: 279209.1022959688, Loss: 9.58779525756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 19.738398893295066, Tokens per sec: 278825.72714854864, Loss: 9.587666511535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 19.55747868200035, Tokens per sec: 276270.0381211446, Loss: 9.584818840026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 19.52081247036728, Tokens per sec: 275752.089164107, Loss: 9.582829475402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 19.74639028129264, Tokens per sec: 278938.61394252797, Loss: 9.581207275390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 19.17239570526971, Tokens per sec: 270830.33444609516, Loss: 9.578659057617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 19.48576330020172, Tokens per sec: 275256.98262531456, Loss: 9.575794219970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 19.733208949624977, Tokens per sec: 278752.41371388274, Loss: 9.573770523071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 19.770401037272013, Tokens per sec: 279277.7912249168, Loss: 9.570661544799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 19.778817937682614, Tokens per sec: 279396.6888310504, Loss: 9.56867790222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 19.78001259918084, Tokens per sec: 279413.56468621676, Loss: 9.565932273864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 19.76507081317531, Tokens per sec: 279202.4961810958, Loss: 9.565648078918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 19.780223290621777, Tokens per sec: 279416.5409252998, Loss: 9.561651229858398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 19.729995944724564, Tokens per sec: 278707.0266268889, Loss: 9.559244155883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 19.64619079494717, Tokens per sec: 277523.19039215695, Loss: 9.556347846984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 19.620910297453186, Tokens per sec: 277166.0766700895, Loss: 9.55431842803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 18.394086758703736, Tokens per sec: 259835.89871979138, Loss: 9.55108642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 19.838626704521513, Tokens per sec: 280241.5508177731, Loss: 9.550041198730469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 19.681760398578337, Tokens per sec: 278025.64860320336, Loss: 9.548714637756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 19.627641626306378, Tokens per sec: 277261.16379809385, Loss: 9.546951293945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 19.28976580198567, Tokens per sec: 272488.3109992714, Loss: 9.543642044067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 19.917930176347078, Tokens per sec: 281361.79609788436, Loss: 9.54166030883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 19.8482582708899, Tokens per sec: 280377.6068631881, Loss: 9.538912773132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 19.788806185324916, Tokens per sec: 279537.78337609733, Loss: 9.538415908813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 19.87032037114435, Tokens per sec: 280689.25732577755, Loss: 9.535465240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 19.778900257797964, Tokens per sec: 279397.8516896058, Loss: 9.533703804016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 19.84191887978944, Tokens per sec: 280288.0562698081, Loss: 9.531012535095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 19.83960023902587, Tokens per sec: 280255.3030206518, Loss: 9.529705047607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 19.757262686975988, Tokens per sec: 279092.1981535322, Loss: 9.529708862304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 19.844043164809726, Tokens per sec: 280318.0640388604, Loss: 9.525237083435059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 19.85538018357429, Tokens per sec: 280478.21140024543, Loss: 9.5263032913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 19.793324787401204, Tokens per sec: 279601.613401847, Loss: 9.520794868469238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 19.829688692474317, Tokens per sec: 280115.29195950425, Loss: 9.518431663513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 19.704786162102156, Tokens per sec: 278350.9118270539, Loss: 9.519503593444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 19.203959179395092, Tokens per sec: 271276.2018475962, Loss: 9.517394065856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 19.874308662351336, Tokens per sec: 280745.59615051607, Loss: 9.51431655883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 19.836217380086637, Tokens per sec: 280207.51656600204, Loss: 9.51163387298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 19.76862574087188, Tokens per sec: 279252.7132886391, Loss: 9.513646125793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 19.84652718802516, Tokens per sec: 280353.15348978504, Loss: 9.508941650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 19.72911924325562, Tokens per sec: 278694.64229288633, Loss: 9.508573532104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 19.80765421171101, Tokens per sec: 279804.03165139206, Loss: 9.504558563232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 19.819801119741722, Tokens per sec: 279975.61955386476, Loss: 9.502758979797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 19.825941393095885, Tokens per sec: 280062.35739881964, Loss: 9.501434326171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 19.841859864062023, Tokens per sec: 280287.2226103406, Loss: 9.498791694641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 19.764635867748144, Tokens per sec: 279196.35211765603, Loss: 9.498726844787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 19.850565623722815, Tokens per sec: 280410.2006584044, Loss: 9.495660781860352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 19.613828770710594, Tokens per sec: 277066.0426271056, Loss: 9.49576187133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 19.8035412601979, Tokens per sec: 279745.9318681943, Loss: 9.494089126586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 19.60506689639354, Tokens per sec: 276942.2719002678, Loss: 9.492534637451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 19.21661201343878, Tokens per sec: 271454.9364892352, Loss: 9.488143920898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 19.909548080591396, Tokens per sec: 281243.39014425356, Loss: 9.488219261169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 19.88469832432158, Tokens per sec: 280892.36109681905, Loss: 9.487029075622559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 19.888943873866438, Tokens per sec: 280952.33396723215, Loss: 9.48667049407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 19.906272578513082, Tokens per sec: 281197.12021862785, Loss: 9.481603622436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 19.908616768876247, Tokens per sec: 281230.23438285757, Loss: 9.479780197143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 19.875669312434393, Tokens per sec: 280764.81676971895, Loss: 9.479792594909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 19.873979919948344, Tokens per sec: 280740.95231693797, Loss: 9.47751522064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 19.766506765167097, Tokens per sec: 279222.78051927465, Loss: 9.476356506347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 19.778165265173225, Tokens per sec: 279387.4691426651, Loss: 9.47391128540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 19.84889797579308, Tokens per sec: 280386.64337044244, Loss: 9.472031593322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 19.896351643285126, Tokens per sec: 281056.97653251217, Loss: 9.471760749816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 19.799113603000933, Tokens per sec: 279683.3865348984, Loss: 9.468032836914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 19.779011711183546, Tokens per sec: 279399.4260863664, Loss: 9.467632293701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 19.282029047989518, Tokens per sec: 272379.02117944317, Loss: 9.464698791503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 19.86754705544361, Tokens per sec: 280650.08131301927, Loss: 9.464622497558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 19.79491803398016, Tokens per sec: 279624.119692165, Loss: 9.462157249450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 19.82178982747458, Tokens per sec: 280003.71215056913, Loss: 9.463635444641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 19.775804706836748, Tokens per sec: 279354.12376352865, Loss: 9.458395957946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 19.792392147805064, Tokens per sec: 279588.4388827299, Loss: 9.456388473510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 19.737336575747463, Tokens per sec: 278810.72079143353, Loss: 9.453662872314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 19.570142349056542, Tokens per sec: 276448.92579052434, Loss: 9.453641891479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 19.912224425618962, Tokens per sec: 281281.1963438569, Loss: 9.454980850219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 19.87304535331383, Tokens per sec: 280727.75057636894, Loss: 9.448933601379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 19.734344454955544, Tokens per sec: 278768.4539257149, Loss: 9.450384140014648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 19.819769507453184, Tokens per sec: 279975.1729969081, Loss: 9.448751449584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 19.736891815100964, Tokens per sec: 278804.43807765626, Loss: 9.443714141845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 19.676719145308688, Tokens per sec: 277954.4355774535, Loss: 9.442527770996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 19.839988048331232, Tokens per sec: 280260.7812365976, Loss: 9.443711280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 19.252557578519628, Tokens per sec: 271962.7055527552, Loss: 9.439813613891602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 19.841078409654365, Tokens per sec: 280276.18374165497, Loss: 9.441360473632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 19.90862052058359, Tokens per sec: 281230.28737968544, Loss: 9.43895435333252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 19.7808684895322, Tokens per sec: 279425.65504120797, Loss: 9.43421459197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 19.923195373963036, Tokens per sec: 281436.17257400014, Loss: 9.434842109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 19.544587024369793, Tokens per sec: 276087.92984415503, Loss: 9.43211841583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 19.806445818535757, Tokens per sec: 279786.9618217877, Loss: 9.433731079101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 19.752279724911347, Tokens per sec: 279021.8085526057, Loss: 9.432369232177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 19.719311467197254, Tokens per sec: 278556.0970995316, Loss: 9.427533149719238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 19.82735748994224, Tokens per sec: 280082.3612621032, Loss: 9.428410530090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 19.709731535152418, Tokens per sec: 278420.7704434601, Loss: 9.42303466796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 19.747536491978188, Tokens per sec: 278954.8053788036, Loss: 9.424752235412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 19.84701983575655, Tokens per sec: 280360.1126592028, Loss: 9.424030303955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 19.764937478251976, Tokens per sec: 279200.61268450855, Loss: 9.42142105102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 19.681552759466044, Tokens per sec: 278022.7154814856, Loss: 9.417797088623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 19.249011807300725, Tokens per sec: 271912.6177901278, Loss: 9.417564392089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 19.777492326115183, Tokens per sec: 279377.96316787973, Loss: 9.414941787719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 19.879723188709203, Tokens per sec: 280822.0820527951, Loss: 9.415337562561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 19.65590117037519, Tokens per sec: 277660.3596987578, Loss: 9.412633895874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 19.784664086104566, Tokens per sec: 279479.2718507564, Loss: 9.409290313720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 19.83322740314248, Tokens per sec: 280165.27998439694, Loss: 9.409199714660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 19.733213808292202, Tokens per sec: 278752.4823476878, Loss: 9.411581993103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 19.863708416659556, Tokens per sec: 280595.85648678034, Loss: 9.408492088317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 19.85233307489489, Tokens per sec: 280435.16777255165, Loss: 9.4042329788208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 19.868484929913485, Tokens per sec: 280663.32978025574, Loss: 9.404372215270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 19.847029401119, Tokens per sec: 280360.24778004794, Loss: 9.402005195617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 19.829043843986817, Tokens per sec: 280106.18279369, Loss: 9.400518417358398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 19.86136510163259, Tokens per sec: 280562.75468759873, Loss: 9.396422386169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 19.862073219771023, Tokens per sec: 280572.7576040422, Loss: 9.396284103393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 19.811939337745073, Tokens per sec: 279864.563581506, Loss: 9.395434379577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 19.244603800074458, Tokens per sec: 271850.3500334174, Loss: 9.394288063049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 19.840017523480697, Tokens per sec: 280261.1976042082, Loss: 9.393020629882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 19.813983743414543, Tokens per sec: 279893.4429703794, Loss: 9.39162826538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 19.847322643930006, Tokens per sec: 280364.39014440344, Loss: 9.387511253356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 19.847542269870196, Tokens per sec: 280367.4925927228, Loss: 9.390684127807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 19.674281774603052, Tokens per sec: 277920.00514249253, Loss: 9.387453079223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 19.801759265074832, Tokens per sec: 279720.75930538157, Loss: 9.386012077331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 19.871691165264792, Tokens per sec: 280708.62124022015, Loss: 9.383606910705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 19.884613086544988, Tokens per sec: 280891.15702321776, Loss: 9.383773803710938 +------------------------------------------------------------------ +[rank0]:[2024-08-03 19:53:30,381] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010123049996764166, 'preprocessing_with_comm': 0.000504554000144708, 'state_converting': 0.03426946099989436, : 0.03610554700026114}) + eval ppl=11914.8408203125, eval loss=9.385540008544922 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600, took 0.21s +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 19.537654586758762, Tokens per sec: 275990.0018425853, Loss: 9.380792617797852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 19.803945298106022, Tokens per sec: 279751.6393302908, Loss: 9.379770278930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 19.89697183870196, Tokens per sec: 281065.73744767084, Loss: 9.378262519836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 19.879400639946557, Tokens per sec: 280817.52571092703, Loss: 9.37590217590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 19.8870176222159, Tokens per sec: 280925.1236286411, Loss: 9.37338924407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 19.02515928999608, Tokens per sec: 268750.4646059248, Loss: 9.374553680419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 19.83146812393705, Tokens per sec: 280140.4283079077, Loss: 9.369978904724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 19.745096079552358, Tokens per sec: 278920.331976333, Loss: 9.369710922241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 19.850323450538827, Tokens per sec: 280406.7797064576, Loss: 9.369688034057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 19.835921154051277, Tokens per sec: 280203.3320604525, Loss: 9.367401123046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 19.824002861844548, Tokens per sec: 280034.97359790065, Loss: 9.367033004760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 19.823800502420326, Tokens per sec: 280032.1150573519, Loss: 9.364569664001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 19.834633315256244, Tokens per sec: 280185.1399775781, Loss: 9.36452865600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 19.78838155008676, Tokens per sec: 279531.7849549644, Loss: 9.36199951171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 19.769158933941572, Tokens per sec: 279260.24520377413, Loss: 9.360067367553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 19.584226044805465, Tokens per sec: 276647.8728646696, Loss: 9.358656883239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 19.81832292713163, Tokens per sec: 279954.7385223482, Loss: 9.358527183532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 19.744420712562103, Tokens per sec: 278910.7917044412, Loss: 9.355177879333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 19.830088099741445, Tokens per sec: 280120.934008907, Loss: 9.352108001708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 19.923480979871897, Tokens per sec: 281440.20705904864, Loss: 9.349259376525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 19.287760021491664, Tokens per sec: 272459.97723178775, Loss: 9.349536895751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 19.849904152461832, Tokens per sec: 280400.8566783618, Loss: 9.348609924316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 19.784443404200157, Tokens per sec: 279476.15448582737, Loss: 9.347586631774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 19.759887572926605, Tokens per sec: 279129.2774393355, Loss: 9.346355438232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 19.87068196327101, Tokens per sec: 280694.3651963902, Loss: 9.3435697555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 19.72830419006383, Tokens per sec: 278683.12880589615, Loss: 9.341896057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 19.724197902785605, Tokens per sec: 278625.1231620533, Loss: 9.339365005493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 19.85739488336619, Tokens per sec: 280506.67116223014, Loss: 9.340113639831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 19.773358669005862, Tokens per sec: 279319.57089627144, Loss: 9.338709831237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 19.848642092948676, Tokens per sec: 280383.0287550656, Loss: 9.336725234985352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 19.82546942831349, Tokens per sec: 280055.6903978967, Loss: 9.332929611206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 19.77085151393784, Tokens per sec: 279284.1546835029, Loss: 9.33400821685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 19.867047573470405, Tokens per sec: 280643.02560271934, Loss: 9.330540657043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 19.84985787831227, Tokens per sec: 280400.203007136, Loss: 9.331522941589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 19.826690454174393, Tokens per sec: 280072.9386775253, Loss: 9.326584815979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 19.176182866975125, Tokens per sec: 270883.83210424107, Loss: 9.326579093933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 19.7277749667198, Tokens per sec: 278675.6529673278, Loss: 9.324836730957031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 19.83457057161703, Tokens per sec: 280184.25365741993, Loss: 9.324392318725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 19.900357773486803, Tokens per sec: 281113.56735188776, Loss: 9.322839736938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 19.759484363117036, Tokens per sec: 279123.5816750055, Loss: 9.320873260498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 19.883568932009165, Tokens per sec: 280876.4072378233, Loss: 9.318032264709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 19.89849743321341, Tokens per sec: 281087.28808109806, Loss: 9.319071769714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 19.909893555247265, Tokens per sec: 281248.270338572, Loss: 9.31443977355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 19.911460542888882, Tokens per sec: 281270.4056936719, Loss: 9.31486988067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 19.894939829469305, Tokens per sec: 281037.03317155753, Loss: 9.310628890991211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 19.912426765522497, Tokens per sec: 281284.0546086554, Loss: 9.313010215759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 19.906761001745426, Tokens per sec: 281204.0197125357, Loss: 9.308375358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 19.908586059532343, Tokens per sec: 281229.80058094737, Loss: 9.308050155639648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 19.831849178268975, Tokens per sec: 280145.8111027208, Loss: 9.308367729187012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 19.763559908161252, Tokens per sec: 279181.1530523306, Loss: 9.3033447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 19.266489740188966, Tokens per sec: 272159.5120480138, Loss: 9.30507755279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 19.7839174292434, Tokens per sec: 279468.72453415935, Loss: 9.301365852355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 19.852860143532617, Tokens per sec: 280442.61317361816, Loss: 9.299999237060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 19.828887963856044, Tokens per sec: 280103.98082224105, Loss: 9.297579765319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 19.83706534174718, Tokens per sec: 280219.4949198632, Loss: 9.298736572265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 19.82651162309225, Tokens per sec: 280070.41249965323, Loss: 9.296517372131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 19.816469890400974, Tokens per sec: 279928.5624218119, Loss: 9.295124053955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 19.828985256699767, Tokens per sec: 280105.3551863951, Loss: 9.291287422180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 19.771151235408432, Tokens per sec: 279288.38856576616, Loss: 9.291452407836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 19.76272728643558, Tokens per sec: 279169.39139124786, Loss: 9.291193962097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 19.903318469227944, Tokens per sec: 281155.39030558086, Loss: 9.287798881530762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 19.900530401000466, Tokens per sec: 281116.00589780445, Loss: 9.282222747802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 19.737889450417708, Tokens per sec: 278818.53072995925, Loss: 9.283228874206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 19.82349758647247, Tokens per sec: 280027.836049724, Loss: 9.281830787658691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 19.262449693650886, Tokens per sec: 272102.44212457247, Loss: 9.279359817504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 19.89406946225987, Tokens per sec: 281024.7383156593, Loss: 9.277012825012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 19.87795538629532, Tokens per sec: 280797.10997698636, Loss: 9.276307106018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 19.896179796792673, Tokens per sec: 281054.5490193448, Loss: 9.275827407836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 19.804731397220046, Tokens per sec: 279762.74381035846, Loss: 9.27474594116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 19.779743352964744, Tokens per sec: 279409.7612991037, Loss: 9.27147102355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 19.780344913271765, Tokens per sec: 279418.25897365843, Loss: 9.272456169128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 19.843358010089954, Tokens per sec: 280308.3855049539, Loss: 9.2673921585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 19.73929437095112, Tokens per sec: 278838.37671602087, Loss: 9.267402648925781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 19.87486583829543, Tokens per sec: 280753.46684907685, Loss: 9.266407012939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 19.849205458882718, Tokens per sec: 280390.9868937707, Loss: 9.264798164367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 19.741378755669565, Tokens per sec: 278867.82085117686, Loss: 9.262993812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 19.781395925364105, Tokens per sec: 279433.10562928, Loss: 9.263589859008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 19.759895824776002, Tokens per sec: 279129.3940054217, Loss: 9.258974075317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 19.301300879601865, Tokens per sec: 272651.2561510761, Loss: 9.256908416748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 19.878029468858276, Tokens per sec: 280798.1564714157, Loss: 9.257883071899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 19.864850243496885, Tokens per sec: 280611.9859965707, Loss: 9.255960464477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 19.857236928496807, Tokens per sec: 280504.43988290755, Loss: 9.25478458404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 19.67730524608337, Tokens per sec: 277962.7148697896, Loss: 9.251411437988281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 19.833739944701833, Tokens per sec: 280172.5201751415, Loss: 9.248675346374512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 19.88185442686726, Tokens per sec: 280852.1880422606, Loss: 9.249220848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 19.78896460348588, Tokens per sec: 279540.0211999028, Loss: 9.24654483795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 19.875668843324654, Tokens per sec: 280764.8101430486, Loss: 9.24455738067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 19.860472023209812, Tokens per sec: 280550.13901183003, Loss: 9.243409156799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 19.80087344174863, Tokens per sec: 279708.24611551303, Loss: 9.241044998168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 19.815029657010083, Tokens per sec: 279908.2176043498, Loss: 9.240296363830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 19.709368523284493, Tokens per sec: 278415.64251750294, Loss: 9.238367080688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 19.700975466362205, Tokens per sec: 278297.0817258193, Loss: 9.23857307434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 19.743949011625908, Tokens per sec: 278904.1284306244, Loss: 9.236348152160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 19.269413307885333, Tokens per sec: 272200.810528869, Loss: 9.23284912109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 19.863027186008562, Tokens per sec: 280586.23338448885, Loss: 9.234176635742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 19.871578664950245, Tokens per sec: 280707.0320544823, Loss: 9.230462074279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 19.836980085183505, Tokens per sec: 280218.29058087454, Loss: 9.22948169708252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 19.736020078029934, Tokens per sec: 278792.12387101643, Loss: 9.228513717651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 19.71980655978484, Tokens per sec: 278563.0908051247, Loss: 9.229036331176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 19.733001945833305, Tokens per sec: 278749.48956673953, Loss: 9.222652435302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 19.811806166824525, Tokens per sec: 279862.6824016313, Loss: 9.221538543701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 19.89772416453128, Tokens per sec: 281076.36484442913, Loss: 9.21997356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 19.869482901675234, Tokens per sec: 280677.42718519963, Loss: 9.218406677246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 19.8228778715551, Tokens per sec: 280019.08192212763, Loss: 9.21884536743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 19.831517581586514, Tokens per sec: 280141.1269494312, Loss: 9.2141695022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 19.862656522834857, Tokens per sec: 280580.9973757582, Loss: 9.215535163879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 19.83340863682827, Tokens per sec: 280167.84010158264, Loss: 9.21342945098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 19.5642037032733, Tokens per sec: 276365.0361479175, Loss: 9.213052749633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 19.86474721291576, Tokens per sec: 280610.5305808171, Loss: 9.206789016723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 19.868998384999053, Tokens per sec: 280670.5828755227, Loss: 9.209085464477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 19.86146513032881, Tokens per sec: 280564.16769855825, Loss: 9.206782341003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 19.832273156241957, Tokens per sec: 280151.8002392891, Loss: 9.204602241516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 19.855778970896107, Tokens per sec: 280483.84469226596, Loss: 9.204571723937988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 19.868941796635898, Tokens per sec: 280669.78350513865, Loss: 9.202629089355469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 19.86987950382857, Tokens per sec: 280683.02960940794, Loss: 9.202109336853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 19.801850677274416, Tokens per sec: 279722.05059922754, Loss: 9.197600364685059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 19.653757568627505, Tokens per sec: 277630.07906053343, Loss: 9.196264266967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 19.880864163788782, Tokens per sec: 280838.19953060785, Loss: 9.194026947021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 19.868373172451687, Tokens per sec: 280661.75108809746, Loss: 9.194561004638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 19.739967704035674, Tokens per sec: 278847.88825684687, Loss: 9.190656661987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 19.70958347540095, Tokens per sec: 278418.6789431268, Loss: 9.18993854522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 19.59230334474527, Tokens per sec: 276761.97325555177, Loss: 9.188608169555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 19.481825939657938, Tokens per sec: 275201.36324997403, Loss: 9.18835163116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 19.832049082479006, Tokens per sec: 280148.63496077654, Loss: 9.18410587310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 19.81212150662198, Tokens per sec: 279867.1369092537, Loss: 9.184896469116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 19.75685678301167, Tokens per sec: 279086.4643314216, Loss: 9.184142112731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 19.846157499657544, Tokens per sec: 280347.9312512197, Loss: 9.180803298950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 19.832977319504145, Tokens per sec: 280161.7472889294, Loss: 9.180522918701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 19.748689964024976, Tokens per sec: 278971.0993894744, Loss: 9.176323890686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 19.835379437784006, Tokens per sec: 280195.6797461515, Loss: 9.17498779296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 19.815420898739365, Tokens per sec: 279913.744306908, Loss: 9.173657417297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 19.856598870971492, Tokens per sec: 280495.426646605, Loss: 9.173521041870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 19.67507083121928, Tokens per sec: 277931.15140040184, Loss: 9.168944358825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 19.718094724799766, Tokens per sec: 278538.9093283468, Loss: 9.167366981506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 19.854668974004788, Tokens per sec: 280468.1648140738, Loss: 9.165474891662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 19.85870097786117, Tokens per sec: 280525.1211261434, Loss: 9.166986465454102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 19.845337600810293, Tokens per sec: 280336.34931422933, Loss: 9.162497520446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 19.259634209053633, Tokens per sec: 272062.67043162265, Loss: 9.16122817993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 19.78427607887517, Tokens per sec: 279473.7908389246, Loss: 9.162940979003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 19.785944972355857, Tokens per sec: 279497.36572160886, Loss: 9.160334587097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 19.844955126530017, Tokens per sec: 280330.94646114635, Loss: 9.157777786254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 19.655292368396527, Tokens per sec: 277651.75972794415, Loss: 9.154230117797852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 19.866587411196235, Tokens per sec: 280636.5253246879, Loss: 9.153287887573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 19.747437708215013, Tokens per sec: 278953.4099538379, Loss: 9.149114608764648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 19.874777815990623, Tokens per sec: 280752.22344107425, Loss: 9.15129566192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 19.88270225359991, Tokens per sec: 280864.1644901227, Loss: 9.149482727050781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 19.83787414048889, Tokens per sec: 280230.92005614156, Loss: 9.145720481872559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 19.882652717212352, Tokens per sec: 280863.46473634045, Loss: 9.1441011428833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 19.838008854126176, Tokens per sec: 280232.8230285192, Loss: 9.145533561706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 19.77777223497846, Tokens per sec: 279381.9171761435, Loss: 9.14245319366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 19.90597516343569, Tokens per sec: 281192.91891660396, Loss: 9.139832496643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 19.855637346136284, Tokens per sec: 280481.8440929846, Loss: 9.140055656433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 19.26560915265294, Tokens per sec: 272147.0728192101, Loss: 9.138922691345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 19.83599535701073, Tokens per sec: 280204.38025560946, Loss: 9.137813568115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 19.87047333592201, Tokens per sec: 280691.41811478534, Loss: 9.136336326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 19.87336250285118, Tokens per sec: 280732.2306484783, Loss: 9.131558418273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 19.83180891767943, Tokens per sec: 280145.24237938033, Loss: 9.135124206542969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 19.84251684207289, Tokens per sec: 280296.5031184807, Loss: 9.132116317749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 19.838069276055105, Tokens per sec: 280233.676552068, Loss: 9.13056755065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 19.85559850195753, Tokens per sec: 280481.29537794215, Loss: 9.12641716003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 19.859109054785176, Tokens per sec: 280530.8856436042, Loss: 9.123394012451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 19.777777485869414, Tokens per sec: 279381.9913505229, Loss: 9.123042106628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 19.702329714783573, Tokens per sec: 278316.21191479097, Loss: 9.119786262512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 19.730615091566644, Tokens per sec: 278715.77272982197, Loss: 9.12409496307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 19.836723656248974, Tokens per sec: 280214.66825139796, Loss: 9.120293617248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 19.789334199564284, Tokens per sec: 279545.24213478557, Loss: 9.117712020874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 19.60302685635047, Tokens per sec: 276913.4541804773, Loss: 9.119959831237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 19.089243651676824, Tokens per sec: 269655.72388460697, Loss: 9.116307258605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 19.880103481040056, Tokens per sec: 280827.45408353844, Loss: 9.113883972167969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 19.884972987392196, Tokens per sec: 280896.24100272224, Loss: 9.11287784576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 19.716956967250674, Tokens per sec: 278522.83730154973, Loss: 9.112260818481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 19.60726570199673, Tokens per sec: 276973.33235124376, Loss: 9.110194206237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 19.59079412598946, Tokens per sec: 276740.653946965, Loss: 9.109123229980469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 19.863655314787383, Tokens per sec: 280595.1063667629, Loss: 9.105875015258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 19.850001109152082, Tokens per sec: 280402.22629399307, Loss: 9.107536315917969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 19.832079637383647, Tokens per sec: 280149.0665810691, Loss: 9.105279922485352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 19.861593484084644, Tokens per sec: 280565.98083089467, Loss: 9.105266571044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 19.87563703896527, Tokens per sec: 280764.36087288836, Loss: 9.101219177246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 19.80914419819079, Tokens per sec: 279825.07928377157, Loss: 9.100358963012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 19.839828161865636, Tokens per sec: 280258.52267143887, Loss: 9.101970672607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 19.893957770981448, Tokens per sec: 281023.16055841104, Loss: 9.098645210266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 19.202599898410444, Tokens per sec: 271257.00056835404, Loss: 9.095633506774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 19.717090518696992, Tokens per sec: 278524.7238567528, Loss: 9.095361709594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 19.697868371497986, Tokens per sec: 278253.1907299225, Loss: 9.093884468078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 19.64389756060171, Tokens per sec: 277490.7960354844, Loss: 9.09261417388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 19.779394239995735, Tokens per sec: 279404.8297097702, Loss: 9.08825397491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 19.8315914064117, Tokens per sec: 280142.1698030424, Loss: 9.088214874267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 19.86219307413626, Tokens per sec: 280574.4506735114, Loss: 9.088967323303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 19.86006423646011, Tokens per sec: 280544.37859338766, Loss: 9.089162826538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 19.804846706090856, Tokens per sec: 279764.37266991916, Loss: 9.083647727966309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 19.853246972264404, Tokens per sec: 280448.0775379268, Loss: 9.085424423217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 19.84640883093752, Tokens per sec: 280351.4815709429, Loss: 9.084223747253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 19.752767805435766, Tokens per sec: 279028.70320540224, Loss: 9.080130577087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 19.67837142103419, Tokens per sec: 277977.7757167983, Loss: 9.084624290466309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 19.82249160321522, Tokens per sec: 280013.6254739464, Loss: 9.083272933959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 19.140764469212364, Tokens per sec: 270383.5098357519, Loss: 9.079702377319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 19.846407631095968, Tokens per sec: 280351.46462191397, Loss: 9.080337524414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 19.716847516769366, Tokens per sec: 278521.2911979269, Loss: 9.074119567871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 19.77208664442779, Tokens per sec: 279301.60220591066, Loss: 9.075111389160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 19.823238165109323, Tokens per sec: 280024.1714490333, Loss: 9.074382781982422 +------------------------------------------------------------------ +[rank0]:[2024-08-03 20:03:18,984] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010223700001006364, 'preprocessing_with_comm': 0.00048214100024779327, 'state_converting': 0.035067328999502934, : 0.03688746199986781}) + eval ppl=8678.255859375, eval loss=9.068575859069824 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800, took 0.18s +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 19.80446810623113, Tokens per sec: 279759.02454711765, Loss: 9.069477081298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 19.787321756638402, Tokens per sec: 279516.8142534165, Loss: 9.070374488830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 19.895777606939127, Tokens per sec: 281048.86766297073, Loss: 9.070639610290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 19.79919549844762, Tokens per sec: 279684.54339456046, Loss: 9.068572998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 19.82792213818141, Tokens per sec: 280090.3375147223, Loss: 9.06838607788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 19.747962631180066, Tokens per sec: 278960.82504501217, Loss: 9.065435409545898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 19.877155310782747, Tokens per sec: 280785.8080655308, Loss: 9.06760311126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 19.85679957302603, Tokens per sec: 280498.2617750569, Loss: 9.063014030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 19.602010139916572, Tokens per sec: 276899.09198724583, Loss: 9.063310623168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 19.807821750396513, Tokens per sec: 279806.3983122374, Loss: 9.061805725097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 19.049075319500954, Tokens per sec: 269088.3037768349, Loss: 9.057160377502441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 19.87265968292877, Tokens per sec: 280722.30257493095, Loss: 9.057899475097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 19.854499997392658, Tokens per sec: 280465.77784099645, Loss: 9.061280250549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 19.852510059123702, Tokens per sec: 280437.6678616703, Loss: 9.059268951416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 19.76722909633942, Tokens per sec: 279232.98420983, Loss: 9.055193901062012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 19.847528066847264, Tokens per sec: 280367.2919600262, Loss: 9.056118965148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 19.734971227725637, Tokens per sec: 278777.30775293376, Loss: 9.051887512207031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 19.887909509135156, Tokens per sec: 280937.7224731644, Loss: 9.052470207214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 19.90058367976958, Tokens per sec: 281116.75851667795, Loss: 9.048672676086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 19.85500638631495, Tokens per sec: 280472.9311192457, Loss: 9.051905632019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 19.86814053073591, Tokens per sec: 280658.46477820387, Loss: 9.049505233764648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 19.789424984865555, Tokens per sec: 279546.5245730308, Loss: 9.049022674560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 19.85583260740595, Tokens per sec: 280484.602364605, Loss: 9.0438232421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 19.899800305502307, Tokens per sec: 281105.69252794795, Loss: 9.046159744262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 19.79778552594758, Tokens per sec: 279664.62604413566, Loss: 9.043682098388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 19.25913835973542, Tokens per sec: 272055.6660364104, Loss: 9.044293403625488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 19.754203078390507, Tokens per sec: 279048.97795146576, Loss: 9.045639991760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 19.782274410028453, Tokens per sec: 279445.5151528006, Loss: 9.041728019714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 19.844659879755824, Tokens per sec: 280326.7757886948, Loss: 9.038419723510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 19.826702881952354, Tokens per sec: 280073.11423301214, Loss: 9.039037704467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 19.824674701764497, Tokens per sec: 280044.46404619986, Loss: 9.037565231323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 19.758112229131275, Tokens per sec: 279104.19883355044, Loss: 9.038818359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 19.87766142120875, Tokens per sec: 280792.9574097259, Loss: 9.034544944763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 19.857276131962085, Tokens per sec: 280504.99367325153, Loss: 9.034258842468262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 19.72338544539978, Tokens per sec: 278613.6463435634, Loss: 9.030887603759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 19.82711728991789, Tokens per sec: 280078.96818311984, Loss: 9.036330223083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 19.808605719486003, Tokens per sec: 279817.4727034593, Loss: 9.031221389770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 19.702015762630978, Tokens per sec: 278311.77700911753, Loss: 9.033121109008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 19.75801984301507, Tokens per sec: 279102.89378210384, Loss: 9.029130935668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 19.874087865843812, Tokens per sec: 280742.47716669703, Loss: 9.029472351074219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 19.268585283312486, Tokens per sec: 272189.11380742415, Loss: 9.029646873474121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 19.756939840745087, Tokens per sec: 279087.637609611, Loss: 9.02568244934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 19.84932211346793, Tokens per sec: 280392.63476296834, Loss: 9.024658203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 19.85754977332284, Tokens per sec: 280508.85914642405, Loss: 9.024413108825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 19.839017666158693, Tokens per sec: 280247.07356373454, Loss: 9.0212984085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 19.86749624949953, Tokens per sec: 280649.3636254106, Loss: 9.02001667022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 19.757770953694028, Tokens per sec: 279099.3779576292, Loss: 9.018159866333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 19.80436317654241, Tokens per sec: 279757.54230446386, Loss: 9.018988609313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 19.799962821368478, Tokens per sec: 279695.3826410729, Loss: 9.018080711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 19.849771853998323, Tokens per sec: 280398.98782286403, Loss: 9.018973350524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 19.778945535164155, Tokens per sec: 279398.4912802139, Loss: 9.016410827636719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 19.805454229576835, Tokens per sec: 279772.95458067366, Loss: 9.013078689575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 19.608290758567623, Tokens per sec: 276987.8123577171, Loss: 9.010915756225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 19.759209775941223, Tokens per sec: 279119.70284119656, Loss: 9.013585090637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 19.24492660889144, Tokens per sec: 271854.9100488276, Loss: 9.011616706848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 19.87014551540163, Tokens per sec: 280686.7873037725, Loss: 9.011697769165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 19.77896381448554, Tokens per sec: 279398.7494949305, Loss: 9.00901985168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 19.798479214504283, Tokens per sec: 279674.42512750014, Loss: 9.007688522338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 19.877322320495537, Tokens per sec: 280788.167254078, Loss: 9.009940147399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 19.87687668191587, Tokens per sec: 280781.8721385678, Loss: 9.004541397094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 19.893473415311888, Tokens per sec: 281016.31852312264, Loss: 9.004328727722168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 19.797097174139985, Tokens per sec: 279654.90234798763, Loss: 9.002721786499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 19.891005817322338, Tokens per sec: 280981.4610958579, Loss: 9.004006385803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 19.89161618005522, Tokens per sec: 280990.0831139729, Loss: 8.999002456665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 19.635668331361796, Tokens per sec: 277374.54948280833, Loss: 9.002236366271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 19.758764915881006, Tokens per sec: 279113.4187230956, Loss: 9.001328468322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 19.754704269993603, Tokens per sec: 279056.0578120932, Loss: 8.997963905334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 19.831628566084074, Tokens per sec: 280142.6947226535, Loss: 8.996589660644531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 19.891947242688158, Tokens per sec: 280994.759723249, Loss: 8.995055198669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 19.0940122724705, Tokens per sec: 269723.08568874723, Loss: 8.994929313659668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 19.90168694897688, Tokens per sec: 281132.3433592293, Loss: 8.993406295776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 19.79459425267928, Tokens per sec: 279619.5459393929, Loss: 8.985540390014648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 19.802885854934033, Tokens per sec: 279736.67357676645, Loss: 8.98991584777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 19.89879340507487, Tokens per sec: 281091.4689961729, Loss: 8.989434242248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 19.895929328894592, Tokens per sec: 281051.01089580264, Loss: 8.994180679321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 19.895115516968197, Tokens per sec: 281039.5149429968, Loss: 8.988174438476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 19.9001726766163, Tokens per sec: 281110.95266313874, Loss: 8.988922119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 19.84737422806448, Tokens per sec: 280365.11882477324, Loss: 8.98474407196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 19.91300531846795, Tokens per sec: 281292.2272799334, Loss: 8.984261512756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 19.760545430564317, Tokens per sec: 279138.5703731335, Loss: 8.98613452911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 19.855441416994722, Tokens per sec: 280479.07638696855, Loss: 8.983353614807129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 19.600413327285334, Tokens per sec: 276876.5353226738, Loss: 8.9828519821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 19.87835416290103, Tokens per sec: 280802.74311763054, Loss: 8.981703758239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 19.270222729606513, Tokens per sec: 272212.2444653903, Loss: 8.978926658630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 19.90005589383072, Tokens per sec: 281109.3029829755, Loss: 8.978094100952148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 19.880740179519268, Tokens per sec: 280836.44812187966, Loss: 8.976689338684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 19.909875708507588, Tokens per sec: 281248.0182345287, Loss: 8.977729797363281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 19.788003532297793, Tokens per sec: 279526.4450545271, Loss: 8.97579288482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 19.718006827666045, Tokens per sec: 278537.66768851795, Loss: 8.973832130432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 19.855206255576604, Tokens per sec: 280475.7544836187, Loss: 8.972662925720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 19.830464343261006, Tokens per sec: 280126.2488459155, Loss: 8.967162132263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 19.86062546772294, Tokens per sec: 280552.3065776078, Loss: 8.969863891601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 19.862953378190902, Tokens per sec: 280585.190771127, Loss: 8.9718599319458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 19.64810805933864, Tokens per sec: 277550.27377622365, Loss: 8.96619987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 19.762690693715562, Tokens per sec: 279168.8744804375, Loss: 8.968031883239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 19.754931871729212, Tokens per sec: 279059.27292694495, Loss: 8.966888427734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 19.7870437269005, Tokens per sec: 279512.88678978285, Loss: 8.968199729919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 19.206899505331915, Tokens per sec: 271317.73705629376, Loss: 8.96349811553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 19.818068730995936, Tokens per sec: 279951.147733513, Loss: 8.963822364807129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 19.84355906802875, Tokens per sec: 280311.2256606467, Loss: 8.96126651763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 19.759293693377142, Tokens per sec: 279120.8882635916, Loss: 8.956876754760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 19.85194834365213, Tokens per sec: 280429.73303749034, Loss: 8.959190368652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 19.895395263692038, Tokens per sec: 281043.46666086983, Loss: 8.959562301635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 19.8977854890643, Tokens per sec: 281077.23111821373, Loss: 8.956215858459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 19.87081657242553, Tokens per sec: 280696.2666928384, Loss: 8.955795288085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 19.88612991923384, Tokens per sec: 280912.5838866487, Loss: 8.953069686889648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 19.906908199028056, Tokens per sec: 281206.09902958595, Loss: 8.949687957763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 19.878364139324447, Tokens per sec: 280802.88404514594, Loss: 8.949759483337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 19.898417235501924, Tokens per sec: 281086.1552037384, Loss: 8.947698593139648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 19.870130484996317, Tokens per sec: 280686.5749834261, Loss: 8.95042896270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 19.920258619654962, Tokens per sec: 281394.68781833013, Loss: 8.949485778808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 19.853177707337842, Tokens per sec: 280447.09909769875, Loss: 8.946601867675781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 19.261453529248502, Tokens per sec: 272088.37025048805, Loss: 8.944866180419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 19.887560569927384, Tokens per sec: 280932.79333839193, Loss: 8.944053649902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 19.84737403146159, Tokens per sec: 280365.11604754976, Loss: 8.945745468139648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 19.8497654527296, Tokens per sec: 280398.8973981839, Loss: 8.941654205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 19.84874151213687, Tokens per sec: 280384.43315608054, Loss: 8.943182945251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 19.883559413035893, Tokens per sec: 280876.2727722743, Loss: 8.9437837600708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 19.824079052046706, Tokens per sec: 280036.0498649592, Loss: 8.936881065368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 19.664299713357913, Tokens per sec: 277778.99798683805, Loss: 8.937646865844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 19.85031017258379, Tokens per sec: 280406.59214132174, Loss: 8.937699317932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 19.839113845938467, Tokens per sec: 280248.432204685, Loss: 8.934187889099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 19.838593826484264, Tokens per sec: 280241.0863807794, Loss: 8.935853958129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 19.84601570030612, Tokens per sec: 280345.92818564764, Loss: 8.931469917297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 19.62625376690174, Tokens per sec: 277241.5588184919, Loss: 8.931201934814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 19.8693692916989, Tokens per sec: 280675.82232431736, Loss: 8.929165840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 19.065527652724647, Tokens per sec: 269320.7103564762, Loss: 8.93012809753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 19.887580214537838, Tokens per sec: 280933.0708392583, Loss: 8.925204277038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 19.678836474738006, Tokens per sec: 277984.34509143856, Loss: 8.930377006530762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 19.87771008270747, Tokens per sec: 280793.64480477944, Loss: 8.925409317016602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 19.862829936343157, Tokens per sec: 280583.4470246791, Loss: 8.922937393188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 19.886690841439208, Tokens per sec: 280920.50750510587, Loss: 8.922538757324219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 19.74908475908189, Tokens per sec: 278976.67628653743, Loss: 8.922476768493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 19.73025598635017, Tokens per sec: 278710.6999894418, Loss: 8.918224334716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 19.874838992363102, Tokens per sec: 280753.08762193477, Loss: 8.92083740234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 19.815097216703524, Tokens per sec: 279909.17195635935, Loss: 8.917098999023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 19.868148227855112, Tokens per sec: 280658.57350814034, Loss: 8.918037414550781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 19.899141007614986, Tokens per sec: 281096.3792491033, Loss: 8.915966033935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 19.91363719912398, Tokens per sec: 281301.1532614348, Loss: 8.914665222167969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 19.809492059192095, Tokens per sec: 279829.9931877391, Loss: 8.912175178527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 19.861948179791767, Tokens per sec: 280570.99128229904, Loss: 8.912226676940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 19.581601708260138, Tokens per sec: 276610.80133979616, Loss: 8.91146183013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 19.90980212517254, Tokens per sec: 281246.9787922208, Loss: 8.9094820022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 19.913272294921704, Tokens per sec: 281295.9986042567, Loss: 8.910167694091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 19.826647812876708, Tokens per sec: 280072.33632416837, Loss: 8.909515380859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 19.762326017472027, Tokens per sec: 279163.7230434173, Loss: 8.905630111694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 19.83831304005531, Tokens per sec: 280237.11997597374, Loss: 8.904086112976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 19.907630304074264, Tokens per sec: 281216.2995258711, Loss: 8.903356552124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 19.843659737448203, Tokens per sec: 280312.64772249846, Loss: 8.903039932250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 19.765736143128603, Tokens per sec: 279211.89466924185, Loss: 8.900912284851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 19.783082673776597, Tokens per sec: 279456.93273173, Loss: 8.89743709564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 19.758935552502003, Tokens per sec: 279115.8291455511, Loss: 8.900238990783691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 19.89588756101276, Tokens per sec: 281050.4208803669, Loss: 8.898756980895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 19.816805154049163, Tokens per sec: 279933.29837486456, Loss: 8.895914077758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 19.907520247446413, Tokens per sec: 281214.74485978833, Loss: 8.893935203552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 19.598551116733983, Tokens per sec: 276850.2296322329, Loss: 8.895049095153809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 19.892290424690614, Tokens per sec: 280999.6075314171, Loss: 8.894309043884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 19.83401369603846, Tokens per sec: 280176.38720183936, Loss: 8.8922119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 19.778886267309748, Tokens per sec: 279397.6540591864, Loss: 8.889449119567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 19.8706477081791, Tokens per sec: 280693.88130704523, Loss: 8.886678695678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 19.910374207992792, Tokens per sec: 281255.06006614823, Loss: 8.888643264770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 19.852914218738178, Tokens per sec: 280443.37704299745, Loss: 8.888532638549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 19.865366607731243, Tokens per sec: 280619.2801866364, Loss: 8.884566307067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 19.854027513151788, Tokens per sec: 280459.10350217397, Loss: 8.88630199432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 19.74450112966016, Tokens per sec: 278911.9276808677, Loss: 8.886926651000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 19.82247039638711, Tokens per sec: 280013.325905106, Loss: 8.881159782409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 19.780229243105616, Tokens per sec: 279416.6250104196, Loss: 8.884156227111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 19.844182130220883, Tokens per sec: 280320.02707203367, Loss: 8.879695892333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 19.822327555809544, Tokens per sec: 280011.3081311152, Loss: 8.881519317626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 19.8896376813404, Tokens per sec: 280962.1347304285, Loss: 8.882111549377441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 19.39066120519406, Tokens per sec: 273913.56511018524, Loss: 8.87584114074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 19.792779387971308, Tokens per sec: 279593.9090589847, Loss: 8.875991821289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 19.657028174904607, Tokens per sec: 277676.27982779726, Loss: 8.877106666564941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 19.80290685789093, Tokens per sec: 279736.9702657107, Loss: 8.87559700012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 19.872782057552143, Tokens per sec: 280724.0312457077, Loss: 8.87519359588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 19.800445987625928, Tokens per sec: 279702.2078746593, Loss: 8.870911598205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 19.839832863169818, Tokens per sec: 280258.58908232476, Loss: 8.871732711791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 19.820840210218194, Tokens per sec: 279990.29780407343, Loss: 8.870229721069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 19.864631402425026, Tokens per sec: 280608.89463534526, Loss: 8.867816925048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 19.849058089836742, Tokens per sec: 280388.9051503818, Loss: 8.867607116699219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 19.70441477264461, Tokens per sec: 278345.66555879696, Loss: 8.867547988891602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 19.852145187342558, Tokens per sec: 280432.5136624749, Loss: 8.867044448852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 19.740970506114554, Tokens per sec: 278862.05389512086, Loss: 8.862549781799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 19.83036686670688, Tokens per sec: 280124.8718866579, Loss: 8.864639282226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 19.880674899129552, Tokens per sec: 280835.525967442, Loss: 8.863824844360352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 19.45732739715734, Tokens per sec: 274855.29546789423, Loss: 8.862114906311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 19.674185867397178, Tokens per sec: 277918.65035193623, Loss: 8.85905647277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 19.86974801252765, Tokens per sec: 280681.1721559341, Loss: 8.857924461364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 19.848765717868584, Tokens per sec: 280384.7750876011, Loss: 8.858853340148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 19.851827527700127, Tokens per sec: 280428.02638459264, Loss: 8.854411125183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 19.868882375454447, Tokens per sec: 280668.94411820476, Loss: 8.855780601501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 19.771981075310805, Tokens per sec: 279300.1109306574, Loss: 8.8565673828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 19.83935485830396, Tokens per sec: 280251.83675884485, Loss: 8.855964660644531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 19.85316047088422, Tokens per sec: 280446.8556145905, Loss: 8.851789474487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 19.804079855074882, Tokens per sec: 279753.5400895615, Loss: 8.85536003112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 19.89614413206411, Tokens per sec: 281054.04521739366, Loss: 8.85122299194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 19.779058073030036, Tokens per sec: 279400.0809964039, Loss: 8.846535682678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 19.685470981179094, Tokens per sec: 278078.06450063223, Loss: 8.85086441040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 19.70353678850694, Tokens per sec: 278333.2631057442, Loss: 8.848158836364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 19.782803826939947, Tokens per sec: 279452.99372571375, Loss: 8.850530624389648 +------------------------------------------------------------------ + eval ppl=6967.75, eval loss=8.849047660827637 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001000, took 0.17s +[rank0]:[2024-08-03 20:13:07,074] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010137780000150087, 'preprocessing_with_comm': 0.0004956900002071052, 'state_converting': 0.027561768999476044, : 0.029389808999439992}) +------------------------------------------------------------------ +iteration: 1001 , TFLOPS: 19.400112844229707, Tokens per sec: 274047.07949203084, Loss: 8.850110054016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1002 , TFLOPS: 19.83371078608067, Tokens per sec: 280172.10827882745, Loss: 8.847855567932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1003 , TFLOPS: 19.831891814867912, Tokens per sec: 280146.413389703, Loss: 8.844223022460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1004 , TFLOPS: 19.82274076651857, Tokens per sec: 280017.14516871044, Loss: 8.844831466674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1005 , TFLOPS: 19.823026076161067, Tokens per sec: 280021.1754686837, Loss: 8.844732284545898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1006 , TFLOPS: 19.73717864789101, Tokens per sec: 278808.4898936971, Loss: 8.843708992004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1007 , TFLOPS: 19.813320600134304, Tokens per sec: 279884.0753712992, Loss: 8.840519905090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1008 , TFLOPS: 19.837307225405613, Tokens per sec: 280222.9117819558, Loss: 8.841193199157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1009 , TFLOPS: 19.838431395426127, Tokens per sec: 280238.791870564, Loss: 8.840511322021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1010 , TFLOPS: 19.84329755594039, Tokens per sec: 280307.53152625466, Loss: 8.84321117401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1011 , TFLOPS: 19.82456812391662, Tokens per sec: 280042.9585215576, Loss: 8.839466094970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1012 , TFLOPS: 19.837584705384625, Tokens per sec: 280226.8314796646, Loss: 8.837504386901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1013 , TFLOPS: 19.858314744189816, Tokens per sec: 280519.66516769177, Loss: 8.834671974182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1014 , TFLOPS: 19.86505088826468, Tokens per sec: 280614.82031578687, Loss: 8.838440895080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1015 , TFLOPS: 19.9013048557211, Tokens per sec: 281126.94588851964, Loss: 8.835386276245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1016 , TFLOPS: 19.403458070093677, Tokens per sec: 274094.3343397538, Loss: 8.832932472229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1017 , TFLOPS: 19.864165410065034, Tokens per sec: 280602.31200119527, Loss: 8.832542419433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1018 , TFLOPS: 19.911597861278956, Tokens per sec: 281272.34546093317, Loss: 8.833185195922852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1019 , TFLOPS: 19.888685582348284, Tokens per sec: 280948.68532679504, Loss: 8.831525802612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1020 , TFLOPS: 19.783367482853187, Tokens per sec: 279460.9559606812, Loss: 8.835027694702148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1021 , TFLOPS: 19.53531642803092, Tokens per sec: 275956.9728815736, Loss: 8.833208084106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1022 , TFLOPS: 19.90703057978017, Tokens per sec: 281207.8277869376, Loss: 8.82816219329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1023 , TFLOPS: 19.813828924261685, Tokens per sec: 279891.2559863638, Loss: 8.826159477233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1024 , TFLOPS: 19.83130873348686, Tokens per sec: 280138.17674949026, Loss: 8.826196670532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1025 , TFLOPS: 19.811182771869653, Tokens per sec: 279853.8762896192, Loss: 8.826947212219238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1026 , TFLOPS: 19.873499139921883, Tokens per sec: 280734.1607913841, Loss: 8.8242769241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1027 , TFLOPS: 19.90009528018123, Tokens per sec: 281109.8593567665, Loss: 8.827529907226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1028 , TFLOPS: 19.82751375196781, Tokens per sec: 280084.5686282194, Loss: 8.83082389831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1029 , TFLOPS: 19.854373003935418, Tokens per sec: 280463.9839243141, Loss: 8.822705268859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1030 , TFLOPS: 19.68772186763062, Tokens per sec: 278109.8606485858, Loss: 8.823699951171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1031 , TFLOPS: 19.131356074160895, Tokens per sec: 270250.6063208463, Loss: 8.817007064819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1032 , TFLOPS: 19.878070760567486, Tokens per sec: 280798.73976041033, Loss: 8.818538665771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1033 , TFLOPS: 19.758690187702012, Tokens per sec: 279112.36310865806, Loss: 8.823715209960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1034 , TFLOPS: 19.863524421930187, Tokens per sec: 280593.2573669386, Loss: 8.821870803833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1035 , TFLOPS: 19.908319911285634, Tokens per sec: 281226.0409559231, Loss: 8.819561004638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1036 , TFLOPS: 19.909241255999003, Tokens per sec: 281239.0559228943, Loss: 8.823446273803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1037 , TFLOPS: 19.916837325691237, Tokens per sec: 281346.358428374, Loss: 8.818158149719238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1038 , TFLOPS: 19.904126891007717, Tokens per sec: 281166.8101168738, Loss: 8.814934730529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1039 , TFLOPS: 19.90612631808825, Tokens per sec: 281195.05413568334, Loss: 8.813552856445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1040 , TFLOPS: 19.89835629548595, Tokens per sec: 281085.2943616631, Loss: 8.8152437210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1041 , TFLOPS: 19.903828694607682, Tokens per sec: 281162.5977778426, Loss: 8.818572044372559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1042 , TFLOPS: 19.89714804583591, Tokens per sec: 281068.22655950394, Loss: 8.812773704528809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1043 , TFLOPS: 19.848741437545, Tokens per sec: 280384.4321023917, Loss: 8.81530475616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1044 , TFLOPS: 19.82913761279622, Tokens per sec: 280107.50737713807, Loss: 8.815464973449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1045 , TFLOPS: 19.837676347070026, Tokens per sec: 280228.12601524004, Loss: 8.813126564025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1046 , TFLOPS: 19.11247006409103, Tokens per sec: 269983.8214859095, Loss: 8.810169219970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1047 , TFLOPS: 19.908998986064443, Tokens per sec: 281235.63360424346, Loss: 8.81233024597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1048 , TFLOPS: 19.865432188336747, Tokens per sec: 280620.20658193895, Loss: 8.815258979797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1049 , TFLOPS: 19.798057118655528, Tokens per sec: 279668.46257792396, Loss: 8.808323860168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1050 , TFLOPS: 19.87168286055401, Tokens per sec: 280708.503927411, Loss: 8.80802059173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1051 , TFLOPS: 19.842566138020043, Tokens per sec: 280297.19947578834, Loss: 8.808692932128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1052 , TFLOPS: 19.869886407307007, Tokens per sec: 280683.1271283306, Loss: 8.809192657470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1053 , TFLOPS: 19.75671280403521, Tokens per sec: 279084.43047634437, Loss: 8.807278633117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1054 , TFLOPS: 19.876033043767443, Tokens per sec: 280769.9548588807, Loss: 8.805764198303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1055 , TFLOPS: 19.905762159632907, Tokens per sec: 281189.9100129681, Loss: 8.808379173278809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1056 , TFLOPS: 19.848348655204745, Tokens per sec: 280378.8836370767, Loss: 8.805094718933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1057 , TFLOPS: 19.842730412158414, Tokens per sec: 280299.5200214582, Loss: 8.804125785827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1058 , TFLOPS: 19.731637795675297, Tokens per sec: 278730.219505282, Loss: 8.803749084472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1059 , TFLOPS: 19.622329731896045, Tokens per sec: 277186.127680448, Loss: 8.808399200439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1060 , TFLOPS: 19.922856097801198, Tokens per sec: 281431.3799399552, Loss: 8.804244041442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1061 , TFLOPS: 19.204973805752942, Tokens per sec: 271290.53451629647, Loss: 8.803339004516602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1062 , TFLOPS: 19.830143918830014, Tokens per sec: 280121.7225124752, Loss: 8.801567077636719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1063 , TFLOPS: 19.70447622110072, Tokens per sec: 278346.5335831261, Loss: 8.80152416229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1064 , TFLOPS: 19.831643585668324, Tokens per sec: 280142.9068901409, Loss: 8.797268867492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1065 , TFLOPS: 19.81647045809662, Tokens per sec: 279928.5704411124, Loss: 8.802127838134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1066 , TFLOPS: 19.876774662067152, Tokens per sec: 280780.4310004767, Loss: 8.802189826965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1067 , TFLOPS: 19.757485000062125, Tokens per sec: 279095.3385606256, Loss: 8.803378105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1068 , TFLOPS: 19.873959907949423, Tokens per sec: 280740.6696263215, Loss: 8.795814514160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1069 , TFLOPS: 19.816824172517695, Tokens per sec: 279933.5670308152, Loss: 8.799450874328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1070 , TFLOPS: 19.848012723408637, Tokens per sec: 280374.13824572915, Loss: 8.798713684082031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1071 , TFLOPS: 19.84077845451395, Tokens per sec: 280271.94655855873, Loss: 8.8002347946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1072 , TFLOPS: 19.706534591583903, Tokens per sec: 278375.6102397393, Loss: 8.796655654907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1073 , TFLOPS: 19.72798719920984, Tokens per sec: 278678.6509753567, Loss: 8.797235488891602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1074 , TFLOPS: 19.679514707466222, Tokens per sec: 277993.92584490514, Loss: 8.793328285217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1075 , TFLOPS: 19.77746094288906, Tokens per sec: 279377.51984667155, Loss: 8.797443389892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1076 , TFLOPS: 19.244665945271137, Tokens per sec: 271851.2278999428, Loss: 8.794412612915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1077 , TFLOPS: 19.61136868295919, Tokens per sec: 277031.291289885, Loss: 8.795763969421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1078 , TFLOPS: 19.831006339310775, Tokens per sec: 280133.9051124397, Loss: 8.795308113098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1079 , TFLOPS: 19.807886127817955, Tokens per sec: 279807.3077112947, Loss: 8.791796684265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1080 , TFLOPS: 19.886538756689117, Tokens per sec: 280918.3591474168, Loss: 8.795698165893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1081 , TFLOPS: 19.90408822505814, Tokens per sec: 281166.2639195067, Loss: 8.79077434539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1082 , TFLOPS: 19.855561096353668, Tokens per sec: 280480.76698428916, Loss: 8.792470932006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1083 , TFLOPS: 19.845182230299578, Tokens per sec: 280334.15454170184, Loss: 8.796072006225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1084 , TFLOPS: 19.885892043050127, Tokens per sec: 280909.2236343682, Loss: 8.79258918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1085 , TFLOPS: 19.73731945269326, Tokens per sec: 278810.47891021176, Loss: 8.791946411132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1086 , TFLOPS: 19.884729701758904, Tokens per sec: 280892.80433625425, Loss: 8.7899808883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1087 , TFLOPS: 19.858961314664533, Tokens per sec: 280528.79865839385, Loss: 8.79068660736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1088 , TFLOPS: 19.8278831520283, Tokens per sec: 280089.7867941422, Loss: 8.790770530700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1089 , TFLOPS: 19.188085157975113, Tokens per sec: 271051.96453285124, Loss: 8.787700653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1090 , TFLOPS: 19.856974309322847, Tokens per sec: 280500.7301097624, Loss: 8.787762641906738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1091 , TFLOPS: 19.880986653077095, Tokens per sec: 280839.92982114793, Loss: 8.786808013916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1092 , TFLOPS: 19.65275576368955, Tokens per sec: 277615.927507928, Loss: 8.79047679901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1093 , TFLOPS: 19.86856736628474, Tokens per sec: 280664.4942810484, Loss: 8.786115646362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1094 , TFLOPS: 19.845067386590838, Tokens per sec: 280332.5322530466, Loss: 8.790481567382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1095 , TFLOPS: 19.87035893318937, Tokens per sec: 280689.8020553831, Loss: 8.791312217712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1096 , TFLOPS: 19.876109967591375, Tokens per sec: 280771.04148912156, Loss: 8.790837287902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1097 , TFLOPS: 19.653288401484932, Tokens per sec: 277623.45157922717, Loss: 8.789687156677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1098 , TFLOPS: 19.86502374284907, Tokens per sec: 280614.4368581271, Loss: 8.789810180664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1099 , TFLOPS: 19.917988983165685, Tokens per sec: 281362.6268062944, Loss: 8.787071228027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1100 , TFLOPS: 19.924565138120478, Tokens per sec: 281455.52193912776, Loss: 8.790165901184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1101 , TFLOPS: 19.852925139634646, Tokens per sec: 280443.53131219203, Loss: 8.78530216217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1102 , TFLOPS: 19.872613839566462, Tokens per sec: 280721.65498903, Loss: 8.790270805358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1103 , TFLOPS: 19.833694097935894, Tokens per sec: 280171.87254116073, Loss: 8.782764434814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1104 , TFLOPS: 19.218477626255698, Tokens per sec: 271481.2902402698, Loss: 8.785906791687012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1105 , TFLOPS: 19.83543788681827, Tokens per sec: 280196.5054004798, Loss: 8.783686637878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1106 , TFLOPS: 19.848198451311834, Tokens per sec: 280376.7618484814, Loss: 8.785788536071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1107 , TFLOPS: 19.856109316674495, Tokens per sec: 280488.51117521466, Loss: 8.784134864807129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1108 , TFLOPS: 19.84866986485367, Tokens per sec: 280383.42106254934, Loss: 8.78343391418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1109 , TFLOPS: 19.85034045827346, Tokens per sec: 280407.01995866856, Loss: 8.782964706420898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1110 , TFLOPS: 19.80639928094506, Tokens per sec: 279786.30442917766, Loss: 8.785778999328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1111 , TFLOPS: 19.748601975145178, Tokens per sec: 278969.85645363526, Loss: 8.782612800598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1112 , TFLOPS: 19.68943930800553, Tokens per sec: 278134.121307414, Loss: 8.784638404846191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1113 , TFLOPS: 19.855761716361478, Tokens per sec: 280483.6009537444, Loss: 8.77921199798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1114 , TFLOPS: 19.857918644287214, Tokens per sec: 280514.0698383054, Loss: 8.781827926635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1115 , TFLOPS: 19.743835725715055, Tokens per sec: 278902.52814750926, Loss: 8.786504745483398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1116 , TFLOPS: 19.858482713405557, Tokens per sec: 280522.0379102314, Loss: 8.783265113830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1117 , TFLOPS: 19.77180505303747, Tokens per sec: 279297.62443017564, Loss: 8.780372619628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1118 , TFLOPS: 19.724062199553646, Tokens per sec: 278623.20621060586, Loss: 8.780960083007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1119 , TFLOPS: 19.17175264038954, Tokens per sec: 270821.2504756178, Loss: 8.781598091125488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1120 , TFLOPS: 19.863089607344087, Tokens per sec: 280587.11515176704, Loss: 8.781340599060059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1121 , TFLOPS: 19.862513190778984, Tokens per sec: 280578.97265911737, Loss: 8.779513359069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1122 , TFLOPS: 19.724996318561868, Tokens per sec: 278636.4016279809, Loss: 8.777416229248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1123 , TFLOPS: 19.727010958183204, Tokens per sec: 278664.8605399928, Loss: 8.77800178527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1124 , TFLOPS: 19.780118260817872, Tokens per sec: 279415.05726841337, Loss: 8.778691291809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1125 , TFLOPS: 19.78381759409313, Tokens per sec: 279467.3142572408, Loss: 8.778544425964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1126 , TFLOPS: 19.776453528230384, Tokens per sec: 279363.2890508373, Loss: 8.778972625732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1127 , TFLOPS: 19.761718663620712, Tokens per sec: 279155.14352893166, Loss: 8.77802562713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1128 , TFLOPS: 19.855986027737156, Tokens per sec: 280486.76958878763, Loss: 8.777835845947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1129 , TFLOPS: 19.892169730116574, Tokens per sec: 280997.90259311127, Loss: 8.779041290283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1130 , TFLOPS: 19.735301359790036, Tokens per sec: 278781.97121694667, Loss: 8.77964973449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1131 , TFLOPS: 19.788582203111364, Tokens per sec: 279534.6193908167, Loss: 8.777162551879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1132 , TFLOPS: 19.7832682190481, Tokens per sec: 279459.5537546167, Loss: 8.776599884033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1133 , TFLOPS: 19.30417114506231, Tokens per sec: 272691.8016815644, Loss: 8.774365425109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1134 , TFLOPS: 19.84678014882831, Tokens per sec: 280356.7268282437, Loss: 8.776046752929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1135 , TFLOPS: 19.91841493845699, Tokens per sec: 281368.64387457195, Loss: 8.776979446411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1136 , TFLOPS: 19.787354080861327, Tokens per sec: 279517.2708671981, Loss: 8.776694297790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1137 , TFLOPS: 19.818808024135148, Tokens per sec: 279961.5910297616, Loss: 8.7746000289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1138 , TFLOPS: 19.68129344673524, Tokens per sec: 278019.05241534143, Loss: 8.775821685791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1139 , TFLOPS: 19.92192220191443, Tokens per sec: 281418.18767440633, Loss: 8.777987480163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1140 , TFLOPS: 19.8754718591979, Tokens per sec: 280762.0275342526, Loss: 8.771953582763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1141 , TFLOPS: 19.87616875208773, Tokens per sec: 280771.8718822061, Loss: 8.77495288848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1142 , TFLOPS: 19.83361472638064, Tokens per sec: 280170.7513341303, Loss: 8.77665901184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1143 , TFLOPS: 19.889217830468574, Tokens per sec: 280956.20389352203, Loss: 8.770686149597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1144 , TFLOPS: 19.82456829301302, Tokens per sec: 280042.96091022284, Loss: 8.77453327178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1145 , TFLOPS: 19.67974731857762, Tokens per sec: 277997.2117224795, Loss: 8.776725769042969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1146 , TFLOPS: 19.912465573051303, Tokens per sec: 281284.6028059786, Loss: 8.773709297180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1147 , TFLOPS: 19.867664931113318, Tokens per sec: 280651.74643132486, Loss: 8.776376724243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1148 , TFLOPS: 19.301897920120652, Tokens per sec: 272659.6899788495, Loss: 8.772956848144531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1149 , TFLOPS: 19.84604201427327, Tokens per sec: 280346.29989821994, Loss: 8.774006843566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1150 , TFLOPS: 19.852133948514783, Tokens per sec: 280432.35490216495, Loss: 8.770501136779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1151 , TFLOPS: 19.86251941016196, Tokens per sec: 280579.0605144692, Loss: 8.77208137512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1152 , TFLOPS: 19.86172049524981, Tokens per sec: 280567.7749977202, Loss: 8.77168083190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1153 , TFLOPS: 19.730382254562386, Tokens per sec: 278712.4836612724, Loss: 8.770306587219238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1154 , TFLOPS: 19.867696037428917, Tokens per sec: 280652.1858408794, Loss: 8.767036437988281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1155 , TFLOPS: 19.833864189706755, Tokens per sec: 280174.2752670327, Loss: 8.77318286895752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1156 , TFLOPS: 19.918874677898483, Tokens per sec: 281375.1381796453, Loss: 8.771383285522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1157 , TFLOPS: 19.92089171199087, Tokens per sec: 281403.6309160892, Loss: 8.77156925201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1158 , TFLOPS: 19.756606948953632, Tokens per sec: 279082.93516153924, Loss: 8.7694673538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1159 , TFLOPS: 19.764719105194356, Tokens per sec: 279197.5279344784, Loss: 8.768357276916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1160 , TFLOPS: 19.848405553482575, Tokens per sec: 280379.68738533277, Loss: 8.767576217651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 19.69611024295274, Tokens per sec: 278228.35530772345, Loss: 8.769164085388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1162 , TFLOPS: 19.44465140348553, Tokens per sec: 274676.23367205256, Loss: 8.770730018615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1163 , TFLOPS: 19.831846409842484, Tokens per sec: 280145.7719957733, Loss: 8.769086837768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1164 , TFLOPS: 19.78573946802518, Tokens per sec: 279494.4627559356, Loss: 8.77014446258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1165 , TFLOPS: 19.60326948798182, Tokens per sec: 276916.8816084772, Loss: 8.7716646194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1166 , TFLOPS: 19.86599591688128, Tokens per sec: 280628.1698429002, Loss: 8.771687507629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1167 , TFLOPS: 19.63282910443393, Tokens per sec: 277334.4424043679, Loss: 8.771112442016602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1168 , TFLOPS: 19.77653431416964, Tokens per sec: 279364.43023753527, Loss: 8.768070220947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1169 , TFLOPS: 19.85479271471799, Tokens per sec: 280469.91278231196, Loss: 8.764915466308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1170 , TFLOPS: 19.864767111295105, Tokens per sec: 280610.811666437, Loss: 8.765798568725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1171 , TFLOPS: 19.865820145607604, Tokens per sec: 280625.68688805366, Loss: 8.76723861694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1172 , TFLOPS: 19.73271428284966, Tokens per sec: 278745.4260233375, Loss: 8.766119003295898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1173 , TFLOPS: 19.6989033242186, Tokens per sec: 278267.81052996044, Loss: 8.770685195922852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1174 , TFLOPS: 19.808877690183618, Tokens per sec: 279821.31457675085, Loss: 8.76999282836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1175 , TFLOPS: 19.65226982144241, Tokens per sec: 277609.06306055596, Loss: 8.766390800476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1176 , TFLOPS: 19.74290378608279, Tokens per sec: 278889.36351612094, Loss: 8.768054008483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1177 , TFLOPS: 19.55069523667543, Tokens per sec: 276174.2147929446, Loss: 8.769559860229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1178 , TFLOPS: 19.824066296148654, Tokens per sec: 280035.86967442965, Loss: 8.766963005065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1179 , TFLOPS: 19.848475693654176, Tokens per sec: 280380.67818932125, Loss: 8.765877723693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1180 , TFLOPS: 19.857553295424783, Tokens per sec: 280508.90889983316, Loss: 8.763923645019531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1181 , TFLOPS: 19.859398853609182, Tokens per sec: 280534.9793580067, Loss: 8.764551162719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1182 , TFLOPS: 19.746055194978364, Tokens per sec: 278933.8804945041, Loss: 8.76561164855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1183 , TFLOPS: 19.694023591290897, Tokens per sec: 278198.87914959807, Loss: 8.76591682434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1184 , TFLOPS: 19.743893958510863, Tokens per sec: 278903.350747241, Loss: 8.764167785644531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1185 , TFLOPS: 19.847955400271076, Tokens per sec: 280373.3284958807, Loss: 8.765645027160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1186 , TFLOPS: 19.84120610032091, Tokens per sec: 280277.987507155, Loss: 8.765681266784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1187 , TFLOPS: 19.80711490925031, Tokens per sec: 279796.4134346577, Loss: 8.765336036682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1188 , TFLOPS: 19.75554048096613, Tokens per sec: 279067.8701750779, Loss: 8.763172149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1189 , TFLOPS: 19.50613561375221, Tokens per sec: 275544.7630663787, Loss: 8.760721206665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1190 , TFLOPS: 19.85050652942255, Tokens per sec: 280409.36588901246, Loss: 8.766097068786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1191 , TFLOPS: 19.71955873934511, Tokens per sec: 278559.5900797273, Loss: 8.765817642211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1192 , TFLOPS: 19.57175663397884, Tokens per sec: 276471.7292696575, Loss: 8.759994506835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1193 , TFLOPS: 19.848727381976943, Tokens per sec: 280384.2335526508, Loss: 8.765172958374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1194 , TFLOPS: 19.83958055361517, Tokens per sec: 280255.0249434388, Loss: 8.763901710510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1195 , TFLOPS: 19.768893843772567, Tokens per sec: 279256.50052521477, Loss: 8.762207984924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1196 , TFLOPS: 19.861768589913225, Tokens per sec: 280568.45438562654, Loss: 8.763066291809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1197 , TFLOPS: 19.621075705682973, Tokens per sec: 277168.4132359982, Loss: 8.76156234741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1198 , TFLOPS: 19.71991169964232, Tokens per sec: 278564.5760166342, Loss: 8.75876235961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1199 , TFLOPS: 19.72124689931626, Tokens per sec: 278583.43712193385, Loss: 8.761147499084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1200 , TFLOPS: 19.75972132794525, Tokens per sec: 279126.92905342724, Loss: 8.762319564819336 +------------------------------------------------------------------ + eval ppl=6370.20556640625, eval loss=8.759387016296387 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001200, took 0.18s +[rank0]:[2024-08-03 20:22:55,421] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0009996830003728974, 'preprocessing_with_comm': 0.000501380999594403, 'state_converting': 0.026713500999903772, : 0.028531369999655}) +------------------------------------------------------------------ +iteration: 1201 , TFLOPS: 19.837959072857753, Tokens per sec: 280232.11981553625, Loss: 8.762863159179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1202 , TFLOPS: 19.85164411923968, Tokens per sec: 280425.4355464184, Loss: 8.764760971069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1203 , TFLOPS: 19.73123607964297, Tokens per sec: 278724.54484213295, Loss: 8.763679504394531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1204 , TFLOPS: 19.854057396372642, Tokens per sec: 280459.5256342238, Loss: 8.761614799499512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1205 , TFLOPS: 19.839870297610275, Tokens per sec: 280259.11788332515, Loss: 8.762256622314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1206 , TFLOPS: 19.735319873572838, Tokens per sec: 278782.2327436784, Loss: 8.765055656433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1207 , TFLOPS: 19.24868317354228, Tokens per sec: 271907.9754912686, Loss: 8.761539459228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1208 , TFLOPS: 19.801746328681688, Tokens per sec: 279720.5765651683, Loss: 8.7612943649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1209 , TFLOPS: 19.666099851453815, Tokens per sec: 277804.42683830013, Loss: 8.76168441772461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1210 , TFLOPS: 19.73532544380098, Tokens per sec: 278782.31142903276, Loss: 8.759488105773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1211 , TFLOPS: 19.718522940078877, Tokens per sec: 278544.9583213386, Loss: 8.757063865661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1212 , TFLOPS: 19.816588694911612, Tokens per sec: 279930.2406609764, Loss: 8.763837814331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1213 , TFLOPS: 19.811424382748612, Tokens per sec: 279857.2892984138, Loss: 8.760769844055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1214 , TFLOPS: 19.778143601045233, Tokens per sec: 279387.163113981, Loss: 8.755441665649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1215 , TFLOPS: 19.74623399349847, Tokens per sec: 278936.4062124031, Loss: 8.75853443145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1216 , TFLOPS: 19.874837014100617, Tokens per sec: 280753.05967688817, Loss: 8.758034706115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1217 , TFLOPS: 19.894551708096344, Tokens per sec: 281031.5505473274, Loss: 8.76072883605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1218 , TFLOPS: 19.812751506965874, Tokens per sec: 279876.0363293607, Loss: 8.759325981140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1219 , TFLOPS: 19.828184158609346, Tokens per sec: 280094.03882994765, Loss: 8.760303497314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1220 , TFLOPS: 19.917806855881423, Tokens per sec: 281360.05406608665, Loss: 8.757594108581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1221 , TFLOPS: 19.771098553253733, Tokens per sec: 279287.64437470125, Loss: 8.760496139526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1222 , TFLOPS: 19.193372396184674, Tokens per sec: 271126.65235562547, Loss: 8.756078720092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1223 , TFLOPS: 19.822500265835085, Tokens per sec: 280013.74784259935, Loss: 8.758166313171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1224 , TFLOPS: 19.84378032524067, Tokens per sec: 280314.35115240177, Loss: 8.755589485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1225 , TFLOPS: 19.86414985223777, Tokens per sec: 280602.0922304569, Loss: 8.759035110473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1226 , TFLOPS: 19.84101099788929, Tokens per sec: 280275.2314792897, Loss: 8.753321647644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1227 , TFLOPS: 19.855903424323266, Tokens per sec: 280485.6027283413, Loss: 8.761347770690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1228 , TFLOPS: 19.868283828525495, Tokens per sec: 280660.4890107972, Loss: 8.753140449523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1229 , TFLOPS: 19.857489314104715, Tokens per sec: 280508.005096126, Loss: 8.758316040039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1230 , TFLOPS: 19.909011237552146, Tokens per sec: 281235.80666944425, Loss: 8.758575439453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1231 , TFLOPS: 19.902389737593346, Tokens per sec: 281142.2709905471, Loss: 8.752603530883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1232 , TFLOPS: 19.88551223018443, Tokens per sec: 280903.85837657645, Loss: 8.753417015075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1233 , TFLOPS: 19.837182822127836, Tokens per sec: 280221.1544542934, Loss: 8.751726150512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1234 , TFLOPS: 19.851862063838286, Tokens per sec: 280428.5142440126, Loss: 8.754291534423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1235 , TFLOPS: 19.81500981084561, Tokens per sec: 279907.93725632003, Loss: 8.757407188415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1236 , TFLOPS: 19.847276124204686, Tokens per sec: 280363.7330041607, Loss: 8.756021499633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1237 , TFLOPS: 19.23208486357958, Tokens per sec: 271673.50683604454, Loss: 8.754698753356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1238 , TFLOPS: 19.826628349347075, Tokens per sec: 280072.0613812598, Loss: 8.757648468017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1239 , TFLOPS: 19.847249169712093, Tokens per sec: 280363.35224349017, Loss: 8.754412651062012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1240 , TFLOPS: 19.854514203608403, Tokens per sec: 280465.9785187949, Loss: 8.75700569152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1241 , TFLOPS: 19.77085154757654, Tokens per sec: 279284.15515868505, Loss: 8.754518508911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1242 , TFLOPS: 19.840557799438294, Tokens per sec: 280268.82957261405, Loss: 8.75235366821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1243 , TFLOPS: 19.828327036244563, Tokens per sec: 280096.0571274168, Loss: 8.757120132446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1244 , TFLOPS: 19.83974602476797, Tokens per sec: 280257.36239820154, Loss: 8.753524780273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1245 , TFLOPS: 19.863447018307713, Tokens per sec: 280592.1639590367, Loss: 8.756263732910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1246 , TFLOPS: 19.891581975332375, Tokens per sec: 280989.5999361442, Loss: 8.752642631530762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1247 , TFLOPS: 19.723430395315297, Tokens per sec: 278614.281308585, Loss: 8.750483512878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1248 , TFLOPS: 19.72378332014523, Tokens per sec: 278619.2667444791, Loss: 8.755431175231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1249 , TFLOPS: 19.89182912637837, Tokens per sec: 280993.09120564826, Loss: 8.749833106994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1250 , TFLOPS: 19.875313664205493, Tokens per sec: 280759.7928629387, Loss: 8.751876831054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1251 , TFLOPS: 19.294164284968804, Tokens per sec: 272550.4442159904, Loss: 8.754141807556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1252 , TFLOPS: 19.675034195971392, Tokens per sec: 277930.6338888404, Loss: 8.755072593688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1253 , TFLOPS: 19.705688469804876, Tokens per sec: 278363.6578761474, Loss: 8.753478050231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1254 , TFLOPS: 19.85537307305947, Tokens per sec: 280478.1109567153, Loss: 8.751415252685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1255 , TFLOPS: 19.8490815437053, Tokens per sec: 280389.23646104126, Loss: 8.753817558288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1256 , TFLOPS: 19.730216733469096, Tokens per sec: 278710.14550104743, Loss: 8.752026557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1257 , TFLOPS: 19.870583866694226, Tokens per sec: 280692.97947865794, Loss: 8.748046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1258 , TFLOPS: 19.78714453797058, Tokens per sec: 279514.3108525993, Loss: 8.752339363098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1259 , TFLOPS: 19.892140304096017, Tokens per sec: 280997.4869194985, Loss: 8.752614974975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1260 , TFLOPS: 19.779444104832052, Tokens per sec: 279405.534103238, Loss: 8.754439353942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1261 , TFLOPS: 19.738517447474802, Tokens per sec: 278827.4018515248, Loss: 8.753114700317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1262 , TFLOPS: 19.875766249039483, Tokens per sec: 280766.18610162614, Loss: 8.750812530517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1263 , TFLOPS: 19.854046881274567, Tokens per sec: 280459.37709736003, Loss: 8.751245498657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1264 , TFLOPS: 19.657518063364254, Tokens per sec: 277683.20001958794, Loss: 8.751198768615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1265 , TFLOPS: 19.842878615612637, Tokens per sec: 280301.6135517447, Loss: 8.750312805175781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1266 , TFLOPS: 19.092751548488152, Tokens per sec: 269705.27663123386, Loss: 8.746441841125488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1267 , TFLOPS: 19.846741291854137, Tokens per sec: 280356.17793245247, Loss: 8.746634483337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1268 , TFLOPS: 19.706536456282745, Tokens per sec: 278375.63658057945, Loss: 8.749066352844238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1269 , TFLOPS: 19.842364178669403, Tokens per sec: 280294.34658670134, Loss: 8.74940299987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1270 , TFLOPS: 19.82522478933364, Tokens per sec: 280052.2346139795, Loss: 8.750382423400879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1271 , TFLOPS: 19.75297076534692, Tokens per sec: 279031.570228463, Loss: 8.754523277282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1272 , TFLOPS: 19.832124833638357, Tokens per sec: 280149.7050258919, Loss: 8.752074241638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1273 , TFLOPS: 19.757254894153668, Tokens per sec: 279092.0880716881, Loss: 8.748509407043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1274 , TFLOPS: 19.843468422371792, Tokens per sec: 280309.94519502483, Loss: 8.750408172607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1275 , TFLOPS: 19.70596049547045, Tokens per sec: 278367.50052591943, Loss: 8.750997543334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1276 , TFLOPS: 19.844511663247218, Tokens per sec: 280324.6820740013, Loss: 8.749045372009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1277 , TFLOPS: 19.721610087827873, Tokens per sec: 278588.5675431697, Loss: 8.750617027282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1278 , TFLOPS: 19.863199730048333, Tokens per sec: 280588.6707512487, Loss: 8.747358322143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1279 , TFLOPS: 19.87395853484045, Tokens per sec: 280740.65022970736, Loss: 8.751696586608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1280 , TFLOPS: 19.264317473088436, Tokens per sec: 272128.8264814113, Loss: 8.747660636901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1281 , TFLOPS: 19.67942183518792, Tokens per sec: 277992.61392590555, Loss: 8.746448516845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1282 , TFLOPS: 19.83547864289421, Tokens per sec: 280197.08112308884, Loss: 8.748787879943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1283 , TFLOPS: 19.680845302069223, Tokens per sec: 278012.7218987151, Loss: 8.745201110839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1284 , TFLOPS: 19.840732189031506, Tokens per sec: 280271.2930097651, Loss: 8.753850936889648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1285 , TFLOPS: 19.888088646280586, Tokens per sec: 280940.2529745037, Loss: 8.749807357788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1286 , TFLOPS: 19.89003200422282, Tokens per sec: 280967.7049575283, Loss: 8.746578216552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1287 , TFLOPS: 19.870754034562577, Tokens per sec: 280695.3832794873, Loss: 8.747658729553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1288 , TFLOPS: 19.85735065765192, Tokens per sec: 280506.0464273159, Loss: 8.750020027160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1289 , TFLOPS: 19.558903088567888, Tokens per sec: 276290.1593680145, Loss: 8.746283531188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1290 , TFLOPS: 19.841116031717956, Tokens per sec: 280276.71519303025, Loss: 8.747945785522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1291 , TFLOPS: 19.70478589480626, Tokens per sec: 278350.90805121703, Loss: 8.750499725341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1292 , TFLOPS: 19.856592322798043, Tokens per sec: 280495.3341467405, Loss: 8.745439529418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1293 , TFLOPS: 19.794091217043867, Tokens per sec: 279612.44002986175, Loss: 8.746336936950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1294 , TFLOPS: 9.417950792135962, Tokens per sec: 133038.49983285996, Loss: 8.747895240783691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1295 , TFLOPS: 18.047944591555133, Tokens per sec: 254946.2750996725, Loss: 8.747198104858398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1296 , TFLOPS: 19.9998968756425, Tokens per sec: 282519.6622782449, Loss: 8.746195793151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1297 , TFLOPS: 20.002172521298267, Tokens per sec: 282551.80817610276, Loss: 8.744499206542969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1298 , TFLOPS: 19.94688111566833, Tokens per sec: 281770.7586865685, Loss: 8.747954368591309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1299 , TFLOPS: 20.000449205727158, Tokens per sec: 282527.4645239242, Loss: 8.740422248840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1300 , TFLOPS: 19.949562877611946, Tokens per sec: 281808.6414058311, Loss: 8.745871543884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1301 , TFLOPS: 19.99668303164294, Tokens per sec: 282474.26333808963, Loss: 8.74658489227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1302 , TFLOPS: 19.94860290304432, Tokens per sec: 281795.0807513774, Loss: 8.743691444396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1303 , TFLOPS: 19.947027941366205, Tokens per sec: 281772.8327545917, Loss: 8.741490364074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1304 , TFLOPS: 20.00753735365454, Tokens per sec: 282627.5920981349, Loss: 8.744733810424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1305 , TFLOPS: 19.999642068669505, Tokens per sec: 282516.0628606877, Loss: 8.74362564086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1306 , TFLOPS: 20.00280006142579, Tokens per sec: 282560.6728430557, Loss: 8.744332313537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1307 , TFLOPS: 19.95353851408159, Tokens per sec: 281864.80146904185, Loss: 8.741119384765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1308 , TFLOPS: 19.978393001598917, Tokens per sec: 282215.8973503434, Loss: 8.743735313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1309 , TFLOPS: 20.05485766027666, Tokens per sec: 283296.0413970943, Loss: 8.740638732910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1310 , TFLOPS: 20.04311206452177, Tokens per sec: 283130.1224542837, Loss: 8.744436264038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1311 , TFLOPS: 20.050409214017836, Tokens per sec: 283233.2023963475, Loss: 8.744409561157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1312 , TFLOPS: 20.04545186881548, Tokens per sec: 283163.1746606509, Loss: 8.742389678955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1313 , TFLOPS: 20.039968784005794, Tokens per sec: 283085.72029784555, Loss: 8.74194622039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1314 , TFLOPS: 20.043838180115728, Tokens per sec: 283140.3796037908, Loss: 8.743453025817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1315 , TFLOPS: 19.682699782607774, Tokens per sec: 278038.9183945627, Loss: 8.742278099060059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1316 , TFLOPS: 20.051135761578877, Tokens per sec: 283243.4656478457, Loss: 8.744284629821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1317 , TFLOPS: 20.04435394144336, Tokens per sec: 283147.6652771623, Loss: 8.739187240600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1318 , TFLOPS: 20.05462088012149, Tokens per sec: 283292.69662737433, Loss: 8.745774269104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1319 , TFLOPS: 20.049064084225922, Tokens per sec: 283214.2010176456, Loss: 8.740020751953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1320 , TFLOPS: 20.02118351858095, Tokens per sec: 282820.35858740105, Loss: 8.737754821777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1321 , TFLOPS: 20.055243955894422, Tokens per sec: 283301.49823060445, Loss: 8.745782852172852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1322 , TFLOPS: 19.923911774697835, Tokens per sec: 281446.29249086324, Loss: 8.738666534423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1323 , TFLOPS: 14.36298192150547, Tokens per sec: 202892.2862454462, Loss: 8.743194580078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1324 , TFLOPS: 19.916692828319793, Tokens per sec: 281344.31725042016, Loss: 8.737385749816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1325 , TFLOPS: 20.0697087829978, Tokens per sec: 283505.82918758737, Loss: 8.741092681884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1326 , TFLOPS: 19.99534708920303, Tokens per sec: 282455.3917404362, Loss: 8.742304801940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1327 , TFLOPS: 19.059956866592756, Tokens per sec: 269242.0171198864, Loss: 8.73961353302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1328 , TFLOPS: 19.99303639857433, Tokens per sec: 282422.75079532986, Loss: 8.739425659179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1329 , TFLOPS: 20.044654731500245, Tokens per sec: 283151.91425433534, Loss: 8.741257667541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1330 , TFLOPS: 20.031611395489037, Tokens per sec: 282967.66336005414, Loss: 8.735894203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1331 , TFLOPS: 19.894998364378726, Tokens per sec: 281037.86003896315, Loss: 8.739197731018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1332 , TFLOPS: 19.937910608510617, Tokens per sec: 281644.04080055037, Loss: 8.74339485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1333 , TFLOPS: 20.040554730757663, Tokens per sec: 283093.9974144467, Loss: 8.738539695739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1334 , TFLOPS: 20.045592320909787, Tokens per sec: 283165.15869479347, Loss: 8.742066383361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1335 , TFLOPS: 20.046176570221, Tokens per sec: 283173.41183325293, Loss: 8.742523193359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1336 , TFLOPS: 20.053558580730886, Tokens per sec: 283277.690526746, Loss: 8.736584663391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1337 , TFLOPS: 20.04355465130128, Tokens per sec: 283136.3744598942, Loss: 8.739542007446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1338 , TFLOPS: 20.051915242970058, Tokens per sec: 283254.47664559033, Loss: 8.737515449523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1339 , TFLOPS: 20.092655109236105, Tokens per sec: 283829.970275898, Loss: 8.737493515014648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1340 , TFLOPS: 20.088512274827767, Tokens per sec: 283771.44836525037, Loss: 8.738142967224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1341 , TFLOPS: 20.090821837985786, Tokens per sec: 283804.07338364277, Loss: 8.738692283630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1342 , TFLOPS: 20.087113828456253, Tokens per sec: 283751.693833562, Loss: 8.740375518798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1343 , TFLOPS: 20.084538228889976, Tokens per sec: 283715.3107699816, Loss: 8.739094734191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1344 , TFLOPS: 20.080015458508193, Tokens per sec: 283651.4218625153, Loss: 8.735730171203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1345 , TFLOPS: 20.042066296352445, Tokens per sec: 283115.3498746121, Loss: 8.731983184814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1346 , TFLOPS: 20.090028603999542, Tokens per sec: 283792.8681159709, Loss: 8.741469383239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1347 , TFLOPS: 19.77544929869119, Tokens per sec: 279349.1032481791, Loss: 8.738492012023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1348 , TFLOPS: 20.0843320150789, Tokens per sec: 283712.3977821485, Loss: 8.736814498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1349 , TFLOPS: 20.093641916026034, Tokens per sec: 283843.9099638253, Loss: 8.740976333618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1350 , TFLOPS: 20.049529992174584, Tokens per sec: 283220.7824593964, Loss: 8.73427963256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1351 , TFLOPS: 20.046035238542625, Tokens per sec: 283171.41537405667, Loss: 8.735564231872559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1352 , TFLOPS: 20.048039366293793, Tokens per sec: 283199.7257948025, Loss: 8.737395286560059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1353 , TFLOPS: 20.021239604630605, Tokens per sec: 282821.15086207655, Loss: 8.732501983642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1354 , TFLOPS: 20.08731516894926, Tokens per sec: 283754.53798063146, Loss: 8.738020896911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1355 , TFLOPS: 20.092599518640718, Tokens per sec: 283829.18500003725, Loss: 8.736956596374512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1356 , TFLOPS: 20.090897411495124, Tokens per sec: 283805.1409392641, Loss: 8.736661911010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1357 , TFLOPS: 20.088692231178037, Tokens per sec: 283773.99043872306, Loss: 8.734790802001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1358 , TFLOPS: 20.089305778027548, Tokens per sec: 283782.65743584774, Loss: 8.735590934753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1359 , TFLOPS: 19.469380065637917, Tokens per sec: 275025.5521372089, Loss: 8.736350059509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1360 , TFLOPS: 20.09660417613372, Tokens per sec: 283885.75501584774, Loss: 8.73475456237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1361 , TFLOPS: 20.028370348076855, Tokens per sec: 282921.88014297, Loss: 8.730506896972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1362 , TFLOPS: 20.044870337837647, Tokens per sec: 283154.9599215208, Loss: 8.735025405883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1363 , TFLOPS: 20.02520655431609, Tokens per sec: 282877.1882152883, Loss: 8.732046127319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1364 , TFLOPS: 20.028361145605913, Tokens per sec: 282921.7501483506, Loss: 8.733315467834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1365 , TFLOPS: 19.995750080615387, Tokens per sec: 282461.0844196749, Loss: 8.737578392028809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1366 , TFLOPS: 19.98207900849351, Tokens per sec: 282267.966089972, Loss: 8.73430347442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1367 , TFLOPS: 19.9556480039944, Tokens per sec: 281894.6002415782, Loss: 8.732553482055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1368 , TFLOPS: 19.981821473807763, Tokens per sec: 282264.32814059185, Loss: 8.734920501708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1369 , TFLOPS: 19.979911088178174, Tokens per sec: 282237.3419263004, Loss: 8.733806610107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1370 , TFLOPS: 20.074255505179092, Tokens per sec: 283570.05643951363, Loss: 8.733080863952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1371 , TFLOPS: 20.0955309401537, Tokens per sec: 283870.5944243454, Loss: 8.73621940612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1372 , TFLOPS: 19.932338682599916, Tokens per sec: 281565.3314633814, Loss: 8.73488998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1373 , TFLOPS: 19.835855747829488, Tokens per sec: 280202.40812850394, Loss: 8.732086181640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1374 , TFLOPS: 20.092824349429932, Tokens per sec: 283832.36097234517, Loss: 8.736315727233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1375 , TFLOPS: 19.96119436423398, Tokens per sec: 281972.94843664643, Loss: 8.733022689819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1376 , TFLOPS: 19.943455360533267, Tokens per sec: 281722.36627785634, Loss: 8.733088493347168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1377 , TFLOPS: 19.395402149171346, Tokens per sec: 273980.5359500686, Loss: 8.73524284362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1378 , TFLOPS: 19.990933543644612, Tokens per sec: 282393.04574893595, Loss: 8.731767654418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1379 , TFLOPS: 19.989514409764666, Tokens per sec: 282372.998984346, Loss: 8.734859466552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1380 , TFLOPS: 19.984743897463098, Tokens per sec: 282305.61046065943, Loss: 8.731040954589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1381 , TFLOPS: 19.994414510510484, Tokens per sec: 282442.2180816466, Loss: 8.734663963317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1382 , TFLOPS: 19.98881726142093, Tokens per sec: 282363.15102783625, Loss: 8.734773635864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1383 , TFLOPS: 19.828171424492616, Tokens per sec: 280093.8589471022, Loss: 8.730626106262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1384 , TFLOPS: 19.933401936273167, Tokens per sec: 281580.35104425973, Loss: 8.73175048828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1385 , TFLOPS: 19.92263990459428, Tokens per sec: 281428.3259826181, Loss: 8.73353385925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1386 , TFLOPS: 19.93428274473107, Tokens per sec: 281592.7933938182, Loss: 8.729631423950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1387 , TFLOPS: 19.961659838622545, Tokens per sec: 281979.5237539031, Loss: 8.733179092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1388 , TFLOPS: 19.83836822813633, Tokens per sec: 280237.899565894, Loss: 8.729873657226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1389 , TFLOPS: 19.929756436041068, Tokens per sec: 281528.854504012, Loss: 8.731574058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1390 , TFLOPS: 19.773106789067935, Tokens per sec: 279316.01282617537, Loss: 8.73200511932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1391 , TFLOPS: 19.868096386216855, Tokens per sec: 280657.84119025775, Loss: 8.728547096252441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1392 , TFLOPS: 19.96125868642895, Tokens per sec: 281973.8570555715, Loss: 8.731868743896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1393 , TFLOPS: 19.483106965526233, Tokens per sec: 275219.4590930641, Loss: 8.737521171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1394 , TFLOPS: 19.92056566782811, Tokens per sec: 281399.0251980037, Loss: 8.727245330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1395 , TFLOPS: 19.93546185740724, Tokens per sec: 281609.44960545417, Loss: 8.732026100158691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1396 , TFLOPS: 19.91011859332392, Tokens per sec: 281251.44923903386, Loss: 8.728912353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1397 , TFLOPS: 19.898986251836668, Tokens per sec: 281094.19316032005, Loss: 8.731281280517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1398 , TFLOPS: 19.90561673865721, Tokens per sec: 281187.8557881289, Loss: 8.730220794677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1399 , TFLOPS: 19.911600481437862, Tokens per sec: 281272.38247344445, Loss: 8.732635498046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1400 , TFLOPS: 19.79474192676935, Tokens per sec: 279621.6319918517, Loss: 8.731203079223633 +------------------------------------------------------------------ + eval ppl=6203.42431640625, eval loss=8.732856750488281 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001400, took 0.25s +[rank0]:[2024-08-03 20:32:46,305] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010135790007552714, 'preprocessing_with_comm': 0.0005530179996640072, 'state_converting': 0.027778276999924856, : 0.029678923000574287}) +------------------------------------------------------------------ +iteration: 1401 , TFLOPS: 19.933827915063503, Tokens per sec: 281586.36844448594, Loss: 8.728950500488281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1402 , TFLOPS: 19.859881150934896, Tokens per sec: 280541.7923170147, Loss: 8.729475021362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1403 , TFLOPS: 19.837379095433473, Tokens per sec: 280223.92702199053, Loss: 8.730966567993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1404 , TFLOPS: 19.898940184709094, Tokens per sec: 281093.54241349845, Loss: 8.732621192932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1405 , TFLOPS: 19.846821351493627, Tokens per sec: 280357.30885939935, Loss: 8.729448318481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1406 , TFLOPS: 19.867038308064583, Tokens per sec: 280642.89471907826, Loss: 8.728209495544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1407 , TFLOPS: 19.55228239248399, Tokens per sec: 276196.6350446992, Loss: 8.730546951293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1408 , TFLOPS: 19.920037961939478, Tokens per sec: 281391.57079509523, Loss: 8.728055953979492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1409 , TFLOPS: 19.91240250662544, Tokens per sec: 281283.71192611824, Loss: 8.729568481445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1410 , TFLOPS: 19.60534315641913, Tokens per sec: 276946.1743648463, Loss: 8.727185249328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1411 , TFLOPS: 19.777043047886245, Tokens per sec: 279371.61663848016, Loss: 8.72789478302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1412 , TFLOPS: 19.91383310459983, Tokens per sec: 281303.9206331478, Loss: 8.726530075073242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1413 , TFLOPS: 19.909604252409277, Tokens per sec: 281244.1836304958, Loss: 8.731109619140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1414 , TFLOPS: 19.92725465941245, Tokens per sec: 281493.51426737936, Loss: 8.726977348327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1415 , TFLOPS: 19.73729171662718, Tokens per sec: 278810.08710899047, Loss: 8.72960376739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1416 , TFLOPS: 19.940687351724794, Tokens per sec: 281683.26523055526, Loss: 8.730692863464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1417 , TFLOPS: 19.936159666617595, Tokens per sec: 281619.30689740274, Loss: 8.727076530456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1418 , TFLOPS: 19.79380872882039, Tokens per sec: 279608.4495854114, Loss: 8.725872039794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1419 , TFLOPS: 19.949680242182037, Tokens per sec: 281810.2993043149, Loss: 8.727147102355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1420 , TFLOPS: 19.950734989350646, Tokens per sec: 281825.19872183277, Loss: 8.723196029663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1421 , TFLOPS: 19.798987094448076, Tokens per sec: 279681.5994680025, Loss: 8.725613594055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1422 , TFLOPS: 19.353835806672052, Tokens per sec: 273393.36747024785, Loss: 8.728713035583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1423 , TFLOPS: 19.947970568332416, Tokens per sec: 281786.14835585724, Loss: 8.72500228881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1424 , TFLOPS: 19.943333777320927, Tokens per sec: 281720.64878659614, Loss: 8.726926803588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1425 , TFLOPS: 19.95414301057651, Tokens per sec: 281873.34062035126, Loss: 8.729510307312012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1426 , TFLOPS: 19.951930148524394, Tokens per sec: 281842.08160719136, Loss: 8.731131553649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1427 , TFLOPS: 19.8493523012197, Tokens per sec: 280393.06119683885, Loss: 8.73210334777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1428 , TFLOPS: 19.95911764526232, Tokens per sec: 281943.6125882585, Loss: 8.72579288482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1429 , TFLOPS: 19.888089633326118, Tokens per sec: 280940.26691756403, Loss: 8.727945327758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1430 , TFLOPS: 19.96114885207806, Tokens per sec: 281972.3055293856, Loss: 8.725200653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1431 , TFLOPS: 19.94688173195146, Tokens per sec: 281770.76739221846, Loss: 8.727843284606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1432 , TFLOPS: 19.946616291195166, Tokens per sec: 281767.01776124345, Loss: 8.723525047302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1433 , TFLOPS: 19.921391753616223, Tokens per sec: 281410.6945320668, Loss: 8.728114128112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1434 , TFLOPS: 19.846103874741132, Tokens per sec: 280347.1737426501, Loss: 8.72605037689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1435 , TFLOPS: 19.938934649489596, Tokens per sec: 281658.50646071543, Loss: 8.72740364074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1436 , TFLOPS: 19.813316836969502, Tokens per sec: 279884.0222126227, Loss: 8.724514961242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1437 , TFLOPS: 19.3692420146368, Tokens per sec: 273610.9964259496, Loss: 8.724143981933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1438 , TFLOPS: 19.954861419417913, Tokens per sec: 281883.48890384054, Loss: 8.728696823120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1439 , TFLOPS: 19.952220806058776, Tokens per sec: 281846.1874517846, Loss: 8.727499008178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1440 , TFLOPS: 19.866298400999355, Tokens per sec: 280632.44275047636, Loss: 8.727423667907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1441 , TFLOPS: 19.79890675911456, Tokens per sec: 279680.4646465864, Loss: 8.727293968200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1442 , TFLOPS: 19.95261268130892, Tokens per sec: 281851.7231034939, Loss: 8.725265502929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1443 , TFLOPS: 19.950798915348575, Tokens per sec: 281826.1017440562, Loss: 8.724550247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1444 , TFLOPS: 19.948523711766956, Tokens per sec: 281793.96209096245, Loss: 8.725043296813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1445 , TFLOPS: 19.94205045366574, Tokens per sec: 281702.52048483986, Loss: 8.72413444519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1446 , TFLOPS: 19.806557278756593, Tokens per sec: 279788.53631510347, Loss: 8.721126556396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1447 , TFLOPS: 19.944540276681323, Tokens per sec: 281737.69186406594, Loss: 8.724739074707031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1448 , TFLOPS: 19.973848879879935, Tokens per sec: 282151.7068326932, Loss: 8.725024223327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1449 , TFLOPS: 19.960344253023646, Tokens per sec: 281960.9397181248, Loss: 8.726920127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1450 , TFLOPS: 19.707896302204382, Tokens per sec: 278394.84584015317, Loss: 8.718846321105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1451 , TFLOPS: 19.269804586885826, Tokens per sec: 272206.33775792236, Loss: 8.722756385803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1452 , TFLOPS: 19.922700201049317, Tokens per sec: 281429.1777337155, Loss: 8.726330757141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1453 , TFLOPS: 19.90751593005935, Tokens per sec: 281214.68387213716, Loss: 8.72843074798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1454 , TFLOPS: 19.926591914728014, Tokens per sec: 281484.1522988858, Loss: 8.725948333740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1455 , TFLOPS: 19.885326626751976, Tokens per sec: 280901.2365321049, Loss: 8.724201202392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1456 , TFLOPS: 19.738065767786225, Tokens per sec: 278821.021398972, Loss: 8.725112915039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1457 , TFLOPS: 19.74021175234422, Tokens per sec: 278851.33569690806, Loss: 8.721070289611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1458 , TFLOPS: 19.875919562838558, Tokens per sec: 280768.35182092997, Loss: 8.722103118896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1459 , TFLOPS: 19.549168855019186, Tokens per sec: 276152.653040266, Loss: 8.72424602508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1460 , TFLOPS: 19.56067563105469, Tokens per sec: 276315.19840235845, Loss: 8.723847389221191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1461 , TFLOPS: 19.884110174016676, Tokens per sec: 280884.0528527044, Loss: 8.725836753845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1462 , TFLOPS: 19.886375219690997, Tokens per sec: 280916.0490146313, Loss: 8.725701332092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1463 , TFLOPS: 19.9048870813395, Tokens per sec: 281177.548608034, Loss: 8.725079536437988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1464 , TFLOPS: 19.93125508662778, Tokens per sec: 281550.0245260507, Loss: 8.720392227172852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1465 , TFLOPS: 19.908654504255505, Tokens per sec: 281230.76743493625, Loss: 8.722765922546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1466 , TFLOPS: 19.911309238985744, Tokens per sec: 281268.2683662705, Loss: 8.72475814819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1467 , TFLOPS: 19.31903829105157, Tokens per sec: 272901.8158176394, Loss: 8.725502967834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1468 , TFLOPS: 19.913016762801472, Tokens per sec: 281292.3889432291, Loss: 8.71988582611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1469 , TFLOPS: 19.868058580243996, Tokens per sec: 280657.30714096985, Loss: 8.719290733337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1470 , TFLOPS: 19.901287331113082, Tokens per sec: 281126.6983349262, Loss: 8.723654747009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1471 , TFLOPS: 19.90376379431038, Tokens per sec: 281161.6809926116, Loss: 8.718539237976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1472 , TFLOPS: 19.889606642437553, Tokens per sec: 280961.6962731502, Loss: 8.722588539123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1473 , TFLOPS: 19.870223840982092, Tokens per sec: 280687.89373530453, Loss: 8.721580505371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1474 , TFLOPS: 19.861906065631327, Tokens per sec: 280570.39637531235, Loss: 8.724864959716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1475 , TFLOPS: 19.871252765110487, Tokens per sec: 280702.42837511154, Loss: 8.724550247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1476 , TFLOPS: 19.879246401893766, Tokens per sec: 280815.34693556355, Loss: 8.723546981811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1477 , TFLOPS: 19.88030235268331, Tokens per sec: 280830.2633554981, Loss: 8.726028442382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1478 , TFLOPS: 19.878920447735737, Tokens per sec: 280810.7424888898, Loss: 8.72087574005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1479 , TFLOPS: 19.879385683960997, Tokens per sec: 280817.31444183824, Loss: 8.71849536895752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1480 , TFLOPS: 19.883570313241215, Tokens per sec: 280876.4267491845, Loss: 8.723073959350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1481 , TFLOPS: 19.280131513736773, Tokens per sec: 272352.21650442004, Loss: 8.720309257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1482 , TFLOPS: 19.77406211367224, Tokens per sec: 279329.50779498706, Loss: 8.723447799682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1483 , TFLOPS: 19.92258415101873, Tokens per sec: 281427.5384044904, Loss: 8.725934028625488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1484 , TFLOPS: 19.93675203566807, Tokens per sec: 281627.67473575333, Loss: 8.72014331817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1485 , TFLOPS: 19.9336520013687, Tokens per sec: 281583.88347779063, Loss: 8.721471786499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1486 , TFLOPS: 19.943724701165674, Tokens per sec: 281726.1709986996, Loss: 8.718544960021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1487 , TFLOPS: 19.927777758600243, Tokens per sec: 281500.903595774, Loss: 8.724493980407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1488 , TFLOPS: 19.88352357580937, Tokens per sec: 280875.7665336073, Loss: 8.721842765808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1489 , TFLOPS: 19.93740570791527, Tokens per sec: 281636.9085464909, Loss: 8.72390365600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1490 , TFLOPS: 19.925453276333513, Tokens per sec: 281468.0678292173, Loss: 8.719475746154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1491 , TFLOPS: 19.915369634046783, Tokens per sec: 281325.6257340857, Loss: 8.719062805175781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1492 , TFLOPS: 19.913059162228492, Tokens per sec: 281292.9878799075, Loss: 8.720664024353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1493 , TFLOPS: 19.92520434613496, Tokens per sec: 281464.5514273046, Loss: 8.717594146728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1494 , TFLOPS: 19.924201534479597, Tokens per sec: 281450.3856537527, Loss: 8.7194185256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1495 , TFLOPS: 19.938625064341647, Tokens per sec: 281654.133243594, Loss: 8.721026420593262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1496 , TFLOPS: 19.62616492332248, Tokens per sec: 277240.30380912044, Loss: 8.720677375793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1497 , TFLOPS: 19.92689621963613, Tokens per sec: 281488.45092704403, Loss: 8.721611022949219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1498 , TFLOPS: 19.73517813207263, Tokens per sec: 278780.23049531586, Loss: 8.71971321105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1499 , TFLOPS: 19.901253433923877, Tokens per sec: 281126.21950133494, Loss: 8.726397514343262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1500 , TFLOPS: 19.81164313385386, Tokens per sec: 279860.37938876584, Loss: 8.720837593078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1501 , TFLOPS: 19.891895882862357, Tokens per sec: 280994.03421147616, Loss: 8.7196626663208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1502 , TFLOPS: 19.887897201920406, Tokens per sec: 280937.54862076027, Loss: 8.719483375549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1503 , TFLOPS: 19.882494822895893, Tokens per sec: 280861.2343123918, Loss: 8.721588134765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1504 , TFLOPS: 19.909091152312868, Tokens per sec: 281236.9355498255, Loss: 8.722280502319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1505 , TFLOPS: 19.823626626758923, Tokens per sec: 280029.6588800304, Loss: 8.716153144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1506 , TFLOPS: 19.88667330852524, Tokens per sec: 280920.25983418216, Loss: 8.719322204589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1507 , TFLOPS: 19.749047136176163, Tokens per sec: 278976.14482326614, Loss: 8.720375061035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1508 , TFLOPS: 19.8868554658459, Tokens per sec: 280922.83299868566, Loss: 8.721713066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1509 , TFLOPS: 19.884787815934875, Tokens per sec: 280893.62526035553, Loss: 8.721094131469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1510 , TFLOPS: 19.898826878190658, Tokens per sec: 281091.94183927943, Loss: 8.716196060180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1511 , TFLOPS: 19.299404409502625, Tokens per sec: 272624.466508345, Loss: 8.712552070617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1512 , TFLOPS: 19.916601260690836, Tokens per sec: 281343.0237609702, Loss: 8.715056419372559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1513 , TFLOPS: 19.775459434568702, Tokens per sec: 279349.2464281519, Loss: 8.715063095092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1514 , TFLOPS: 19.768144368405885, Tokens per sec: 279245.9133942512, Loss: 8.716140747070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1515 , TFLOPS: 19.872859717459846, Tokens per sec: 280725.12827390904, Loss: 8.719261169433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1516 , TFLOPS: 19.864199832953332, Tokens per sec: 280602.7982608414, Loss: 8.716317176818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1517 , TFLOPS: 19.874504848460777, Tokens per sec: 280748.36748647486, Loss: 8.718362808227539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1518 , TFLOPS: 19.93039496543791, Tokens per sec: 281537.8744059981, Loss: 8.713003158569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1519 , TFLOPS: 19.92371055398459, Tokens per sec: 281443.4500358094, Loss: 8.720197677612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1520 , TFLOPS: 19.81049493353857, Tokens per sec: 279844.1598468692, Loss: 8.716005325317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1521 , TFLOPS: 19.935412433957353, Tokens per sec: 281608.75144703576, Loss: 8.717846870422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1522 , TFLOPS: 19.925834083511457, Tokens per sec: 281473.4471327194, Loss: 8.714762687683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1523 , TFLOPS: 19.923646111468926, Tokens per sec: 281442.53971722757, Loss: 8.716044425964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1524 , TFLOPS: 19.888560397683044, Tokens per sec: 280946.91696120973, Loss: 8.718062400817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1525 , TFLOPS: 19.877026755814967, Tokens per sec: 280783.9920908632, Loss: 8.713427543640137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1526 , TFLOPS: 19.269910517216264, Tokens per sec: 272207.83413569705, Loss: 8.714560508728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1527 , TFLOPS: 19.77471471057229, Tokens per sec: 279338.72641531064, Loss: 8.71949291229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1528 , TFLOPS: 19.886977430004233, Tokens per sec: 280924.5558712103, Loss: 8.71689224243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1529 , TFLOPS: 19.894346754672533, Tokens per sec: 281028.6553637953, Loss: 8.715592384338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1530 , TFLOPS: 19.89099068044432, Tokens per sec: 280981.24727147206, Loss: 8.71645450592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1531 , TFLOPS: 19.906105832116186, Tokens per sec: 281194.7647496957, Loss: 8.717512130737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1532 , TFLOPS: 19.889790991334106, Tokens per sec: 280964.3003959774, Loss: 8.716859817504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1533 , TFLOPS: 19.88943226407327, Tokens per sec: 280959.23299461976, Loss: 8.715392112731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1534 , TFLOPS: 19.890295627909637, Tokens per sec: 280971.42892047827, Loss: 8.718549728393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1535 , TFLOPS: 19.805315002719503, Tokens per sec: 279770.9878542971, Loss: 8.714948654174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1536 , TFLOPS: 19.910383452459048, Tokens per sec: 281255.19065399584, Loss: 8.716691970825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1537 , TFLOPS: 19.897109863504017, Tokens per sec: 281067.6871937473, Loss: 8.714335441589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1538 , TFLOPS: 19.906299898157968, Tokens per sec: 281197.50613746006, Loss: 8.714554786682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1539 , TFLOPS: 19.92053957928892, Tokens per sec: 281398.6566698394, Loss: 8.716383934020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1540 , TFLOPS: 19.67043618041391, Tokens per sec: 277865.6820638121, Loss: 8.718799591064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1541 , TFLOPS: 19.318035481647236, Tokens per sec: 272887.65007588564, Loss: 8.714471817016602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1542 , TFLOPS: 19.786482994584418, Tokens per sec: 279504.9658537124, Loss: 8.715974807739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1543 , TFLOPS: 19.64948499073135, Tokens per sec: 277569.7243861176, Loss: 8.709980010986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1544 , TFLOPS: 19.9173798205763, Tokens per sec: 281354.02174147347, Loss: 8.712747573852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1545 , TFLOPS: 19.91706767520581, Tokens per sec: 281349.6123585051, Loss: 8.714183807373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1546 , TFLOPS: 19.734659538819734, Tokens per sec: 278772.9048180097, Loss: 8.714799880981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1547 , TFLOPS: 19.842191955758437, Tokens per sec: 280291.91375622497, Loss: 8.71054744720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1548 , TFLOPS: 19.84321250331612, Tokens per sec: 280306.33006812545, Loss: 8.715509414672852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1549 , TFLOPS: 19.801930489885073, Tokens per sec: 279723.17803663126, Loss: 8.715847969055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1550 , TFLOPS: 19.802334907480112, Tokens per sec: 279728.89086220635, Loss: 8.713484764099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1551 , TFLOPS: 19.85727135452062, Tokens per sec: 280504.92618684605, Loss: 8.71259593963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1552 , TFLOPS: 19.87206932216576, Tokens per sec: 280713.96310576145, Loss: 8.715865135192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1553 , TFLOPS: 19.741584895619432, Tokens per sec: 278870.7327956424, Loss: 8.714841842651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1554 , TFLOPS: 19.857761224009735, Tokens per sec: 280511.84611065796, Loss: 8.713411331176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1555 , TFLOPS: 19.86608420103268, Tokens per sec: 280629.4169497625, Loss: 8.716757774353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1556 , TFLOPS: 19.27041287135983, Tokens per sec: 272214.9304184363, Loss: 8.710882186889648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1557 , TFLOPS: 19.86144261122466, Tokens per sec: 280563.8495924331, Loss: 8.718244552612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1558 , TFLOPS: 19.933688970068424, Tokens per sec: 281584.40569971135, Loss: 8.714912414550781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1559 , TFLOPS: 19.768079004874476, Tokens per sec: 279244.9900653493, Loss: 8.715063095092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1560 , TFLOPS: 19.632870929378555, Tokens per sec: 277335.0332258758, Loss: 8.715388298034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1561 , TFLOPS: 19.7277798093219, Tokens per sec: 278675.721374196, Loss: 8.715615272521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1562 , TFLOPS: 19.868614067398, Tokens per sec: 280665.15398358734, Loss: 8.714966773986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1563 , TFLOPS: 19.7459768319841, Tokens per sec: 278932.7735344626, Loss: 8.71033763885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1564 , TFLOPS: 19.821557267876724, Tokens per sec: 280000.4270006779, Loss: 8.713359832763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1565 , TFLOPS: 19.748602075822067, Tokens per sec: 278969.85787580267, Loss: 8.71481990814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1566 , TFLOPS: 19.88862151597852, Tokens per sec: 280947.7803216712, Loss: 8.713007926940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1567 , TFLOPS: 19.890000446652838, Tokens per sec: 280967.25917352905, Loss: 8.710655212402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1568 , TFLOPS: 19.865904230709987, Tokens per sec: 280626.87467891455, Loss: 8.71265983581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1569 , TFLOPS: 19.74515950641007, Tokens per sec: 278921.22794767376, Loss: 8.711642265319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1570 , TFLOPS: 19.804924085701703, Tokens per sec: 279765.46573863144, Loss: 8.713985443115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1571 , TFLOPS: 18.978587895023914, Tokens per sec: 268092.594474835, Loss: 8.714831352233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1572 , TFLOPS: 19.930920485578653, Tokens per sec: 281545.29793290957, Loss: 8.715216636657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1573 , TFLOPS: 19.799950454078104, Tokens per sec: 279695.2079400372, Loss: 8.711968421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1574 , TFLOPS: 19.81062439223209, Tokens per sec: 279845.9885876172, Loss: 8.71446704864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1575 , TFLOPS: 19.87086626730411, Tokens per sec: 280696.9686854737, Loss: 8.714821815490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1576 , TFLOPS: 19.86687784267612, Tokens per sec: 280640.62797602266, Loss: 8.711584091186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1577 , TFLOPS: 19.85230268137532, Tokens per sec: 280434.7384319937, Loss: 8.713045120239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1578 , TFLOPS: 19.836860182752968, Tokens per sec: 280216.5968324321, Loss: 8.714217185974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1579 , TFLOPS: 19.86927406170739, Tokens per sec: 280674.47710012906, Loss: 8.711942672729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1580 , TFLOPS: 19.72216204832711, Tokens per sec: 278596.36456806463, Loss: 8.705574035644531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1581 , TFLOPS: 19.86870424995113, Tokens per sec: 280666.4279073786, Loss: 8.713763236999512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1582 , TFLOPS: 19.875823800931464, Tokens per sec: 280766.99908287235, Loss: 8.714784622192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1583 , TFLOPS: 19.73917168346168, Tokens per sec: 278836.64362568053, Loss: 8.714305877685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1584 , TFLOPS: 19.75576035734863, Tokens per sec: 279070.9761611594, Loss: 8.709047317504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1585 , TFLOPS: 19.908765999093784, Tokens per sec: 281232.3424172601, Loss: 8.710515022277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1586 , TFLOPS: 19.317757888899774, Tokens per sec: 272883.7287852034, Loss: 8.71303939819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1587 , TFLOPS: 19.898934112858164, Tokens per sec: 281093.45664219244, Loss: 8.711847305297852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1588 , TFLOPS: 19.905751030505535, Tokens per sec: 281189.7528022922, Loss: 8.7117919921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1589 , TFLOPS: 19.903846939590654, Tokens per sec: 281162.8555074929, Loss: 8.713041305541992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1590 , TFLOPS: 19.90037083222699, Tokens per sec: 281113.7518203823, Loss: 8.710803031921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1591 , TFLOPS: 19.744339350126957, Tokens per sec: 278909.64237413, Loss: 8.713371276855469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1592 , TFLOPS: 19.853849613194477, Tokens per sec: 280456.59047742333, Loss: 8.713919639587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1593 , TFLOPS: 19.88731526745317, Tokens per sec: 280929.32818191627, Loss: 8.712091445922852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1594 , TFLOPS: 19.792893728421664, Tokens per sec: 279595.5242385839, Loss: 8.710397720336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1595 , TFLOPS: 19.738622947873857, Tokens per sec: 278828.8921560647, Loss: 8.711915969848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1596 , TFLOPS: 19.73138111470122, Tokens per sec: 278726.5936154807, Loss: 8.709060668945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1597 , TFLOPS: 19.864823988283774, Tokens per sec: 280611.61511396134, Loss: 8.710445404052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1598 , TFLOPS: 19.651795840203405, Tokens per sec: 277602.3675750541, Loss: 8.708422660827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1599 , TFLOPS: 19.84614221405626, Tokens per sec: 280347.7153259607, Loss: 8.707646369934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1600 , TFLOPS: 19.11850650914789, Tokens per sec: 270069.0926465274, Loss: 8.711454391479492 +------------------------------------------------------------------ +[rank0]:[2024-08-03 20:42:33,855] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010116060002474114, 'preprocessing_with_comm': 0.0006756760003554518, 'state_converting': 0.029059769000014057, : 0.031082213999980013}) + eval ppl=6086.47412109375, eval loss=8.713824272155762 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001600, took 0.19s +------------------------------------------------------------------ +iteration: 1601 , TFLOPS: 19.78636086522287, Tokens per sec: 279503.24064751796, Loss: 8.715150833129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1602 , TFLOPS: 19.782398576813247, Tokens per sec: 279447.2691397499, Loss: 8.70814037322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1603 , TFLOPS: 19.835555639095443, Tokens per sec: 280198.1687757355, Loss: 8.71230411529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1604 , TFLOPS: 19.863341686796268, Tokens per sec: 280590.6760402126, Loss: 8.70943832397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1605 , TFLOPS: 19.703416602978095, Tokens per sec: 278331.5653582392, Loss: 8.708857536315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1606 , TFLOPS: 19.711921516177068, Tokens per sec: 278451.7062379458, Loss: 8.709183692932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1607 , TFLOPS: 19.85934888347538, Tokens per sec: 280534.27347710065, Loss: 8.71059513092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1608 , TFLOPS: 19.836253141794383, Tokens per sec: 280208.0217378866, Loss: 8.712084770202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1609 , TFLOPS: 19.85397089471096, Tokens per sec: 280458.303706911, Loss: 8.710733413696289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1610 , TFLOPS: 19.77321243749988, Tokens per sec: 279317.50522183615, Loss: 8.712315559387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1611 , TFLOPS: 19.849920691582863, Tokens per sec: 280401.09031091083, Loss: 8.707996368408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1612 , TFLOPS: 19.85665545690553, Tokens per sec: 280496.2259826752, Loss: 8.713105201721191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1613 , TFLOPS: 19.582182123214473, Tokens per sec: 276619.00031391584, Loss: 8.712312698364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1614 , TFLOPS: 19.70094947555794, Tokens per sec: 278296.7145782641, Loss: 8.71127700805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1615 , TFLOPS: 19.54483311590518, Tokens per sec: 276091.406146953, Loss: 8.712800979614258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1616 , TFLOPS: 19.92335398369957, Tokens per sec: 281438.4131040128, Loss: 8.710052490234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1617 , TFLOPS: 19.77548989612422, Tokens per sec: 279349.6767297895, Loss: 8.712725639343262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1618 , TFLOPS: 19.91634907007015, Tokens per sec: 281339.46130215214, Loss: 8.708104133605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1619 , TFLOPS: 19.6586758681895, Tokens per sec: 277699.55523532967, Loss: 8.714773178100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1620 , TFLOPS: 19.780888011468935, Tokens per sec: 279425.93080917856, Loss: 8.708724975585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1621 , TFLOPS: 19.898257509813323, Tokens per sec: 281083.8989097245, Loss: 8.712664604187012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1622 , TFLOPS: 19.749848773328626, Tokens per sec: 278987.46879453416, Loss: 8.71043872833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1623 , TFLOPS: 19.803757879859784, Tokens per sec: 279748.9918496582, Loss: 8.706063270568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1624 , TFLOPS: 19.860076767206987, Tokens per sec: 280544.55560341914, Loss: 8.71303653717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1625 , TFLOPS: 19.821666078556273, Tokens per sec: 280001.9640664253, Loss: 8.705671310424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1626 , TFLOPS: 19.741209300779325, Tokens per sec: 278865.42712191615, Loss: 8.707063674926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1627 , TFLOPS: 19.6766627940014, Tokens per sec: 277953.6395557338, Loss: 8.714202880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1628 , TFLOPS: 19.732213616734345, Tokens per sec: 278738.3535857799, Loss: 8.71171760559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1629 , TFLOPS: 19.78387555103273, Tokens per sec: 279468.13296021236, Loss: 8.70943546295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1630 , TFLOPS: 19.554691785535486, Tokens per sec: 276230.6702657525, Loss: 8.714376449584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1631 , TFLOPS: 19.85360480922, Tokens per sec: 280453.1323627829, Loss: 8.711129188537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1632 , TFLOPS: 19.613312410797867, Tokens per sec: 277058.74849808763, Loss: 8.704886436462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1633 , TFLOPS: 19.84204890534909, Tokens per sec: 280289.89301813877, Loss: 8.70971965789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1634 , TFLOPS: 19.862165793949693, Tokens per sec: 280574.0653120696, Loss: 8.710208892822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1635 , TFLOPS: 19.838694879033234, Tokens per sec: 280242.51385474013, Loss: 8.705789566040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1636 , TFLOPS: 19.840803426718423, Tokens per sec: 280272.2993173163, Loss: 8.706201553344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1637 , TFLOPS: 19.723858233099122, Tokens per sec: 278620.32496905717, Loss: 8.70978832244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1638 , TFLOPS: 19.78894727616758, Tokens per sec: 279539.77643323503, Loss: 8.708168029785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1639 , TFLOPS: 19.79536498207159, Tokens per sec: 279630.43330591166, Loss: 8.709214210510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1640 , TFLOPS: 19.582341031926667, Tokens per sec: 276621.2450672754, Loss: 8.705278396606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1641 , TFLOPS: 19.458517610250308, Tokens per sec: 274872.10848463903, Loss: 8.709809303283691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1642 , TFLOPS: 19.783377660530572, Tokens per sec: 279461.0997311214, Loss: 8.705561637878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1643 , TFLOPS: 19.761364016987013, Tokens per sec: 279150.13377074123, Loss: 8.709878921508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1644 , TFLOPS: 19.721047294912523, Tokens per sec: 278580.61749895866, Loss: 8.710471153259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1645 , TFLOPS: 19.080215223579817, Tokens per sec: 269528.18780415965, Loss: 8.710009574890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1646 , TFLOPS: 19.822984769362566, Tokens per sec: 280020.59196653694, Loss: 8.710027694702148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1647 , TFLOPS: 19.873118991456597, Tokens per sec: 280728.7907928938, Loss: 8.709575653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1648 , TFLOPS: 19.850603024076413, Tokens per sec: 280410.7289778919, Loss: 8.708860397338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1649 , TFLOPS: 19.760319990983156, Tokens per sec: 279135.38580099645, Loss: 8.710758209228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1650 , TFLOPS: 19.70859206186551, Tokens per sec: 278404.6741800546, Loss: 8.708826065063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1651 , TFLOPS: 19.776508756343187, Tokens per sec: 279364.0692062488, Loss: 8.708502769470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1652 , TFLOPS: 19.843256564432277, Tokens per sec: 280306.95247791754, Loss: 8.705910682678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1653 , TFLOPS: 19.872723333860208, Tokens per sec: 280723.2017115498, Loss: 8.706013679504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1654 , TFLOPS: 19.872617611696533, Tokens per sec: 280721.70827435044, Loss: 8.708366394042969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1655 , TFLOPS: 19.846437464274644, Tokens per sec: 280351.8860470651, Loss: 8.710007667541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1656 , TFLOPS: 19.594585373936212, Tokens per sec: 276794.20932758466, Loss: 8.708097457885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1657 , TFLOPS: 19.862165019950538, Tokens per sec: 280574.05437851424, Loss: 8.704961776733398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1658 , TFLOPS: 19.866451487255897, Tokens per sec: 280634.6052555016, Loss: 8.708113670349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1659 , TFLOPS: 19.764070199745564, Tokens per sec: 279188.36145980196, Loss: 8.70886516571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1660 , TFLOPS: 19.200430656697748, Tokens per sec: 271226.35773854936, Loss: 8.706459999084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1661 , TFLOPS: 19.867010851541398, Tokens per sec: 280642.50686669553, Loss: 8.701784133911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1662 , TFLOPS: 19.549522503589966, Tokens per sec: 276157.6486997639, Loss: 8.704561233520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1663 , TFLOPS: 19.872820567979108, Tokens per sec: 280724.5752461537, Loss: 8.708252906799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1664 , TFLOPS: 19.761846692508723, Tokens per sec: 279156.952072167, Loss: 8.709460258483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1665 , TFLOPS: 19.863713781191713, Tokens per sec: 280595.9322664617, Loss: 8.70559024810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1666 , TFLOPS: 19.793749594363852, Tokens per sec: 279607.61424876977, Loss: 8.71082878112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1667 , TFLOPS: 19.870212554508083, Tokens per sec: 280687.73430194124, Loss: 8.704258918762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1668 , TFLOPS: 19.87863238434091, Tokens per sec: 280806.6732892571, Loss: 8.708089828491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1669 , TFLOPS: 19.865832221756072, Tokens per sec: 280625.8574764026, Loss: 8.704144477844238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1670 , TFLOPS: 19.80833464761988, Tokens per sec: 279813.64352711174, Loss: 8.707871437072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1671 , TFLOPS: 19.76802342703739, Tokens per sec: 279244.204969713, Loss: 8.704421043395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1672 , TFLOPS: 19.73447063475018, Tokens per sec: 278770.23634855385, Loss: 8.704329490661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1673 , TFLOPS: 19.85370453590759, Tokens per sec: 280454.54110755154, Loss: 8.70632266998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1674 , TFLOPS: 19.795864595679635, Tokens per sec: 279637.4908756928, Loss: 8.707329750061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1675 , TFLOPS: 19.144704735710935, Tokens per sec: 270439.1702607723, Loss: 8.706721305847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1676 , TFLOPS: 19.858020083827785, Tokens per sec: 280515.50277893123, Loss: 8.706640243530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1677 , TFLOPS: 19.85719254062181, Tokens per sec: 280503.8128573018, Loss: 8.70490837097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1678 , TFLOPS: 19.85741033560785, Tokens per sec: 280506.8894414604, Loss: 8.708283424377441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1679 , TFLOPS: 19.82891435422378, Tokens per sec: 280104.3536140523, Loss: 8.705379486083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1680 , TFLOPS: 19.842743375045703, Tokens per sec: 280299.70313592936, Loss: 8.710485458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1681 , TFLOPS: 19.840547102160823, Tokens per sec: 280268.6784622741, Loss: 8.711043357849121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1682 , TFLOPS: 19.822700630817465, Tokens per sec: 280016.578209551, Loss: 8.703615188598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1683 , TFLOPS: 19.71368978575721, Tokens per sec: 278476.6849129712, Loss: 8.706537246704102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1684 , TFLOPS: 19.81636115703388, Tokens per sec: 279927.0264481846, Loss: 8.70304012298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1685 , TFLOPS: 19.803099753844087, Tokens per sec: 279739.6951247377, Loss: 8.707354545593262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1686 , TFLOPS: 19.885689493437482, Tokens per sec: 280906.3624072071, Loss: 8.706080436706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1687 , TFLOPS: 19.856138636185598, Tokens per sec: 280488.9253442689, Loss: 8.705055236816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1688 , TFLOPS: 19.865580323596195, Tokens per sec: 280622.29914890224, Loss: 8.70305347442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1689 , TFLOPS: 19.256293747344778, Tokens per sec: 272015.4828826214, Loss: 8.704776763916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1690 , TFLOPS: 19.799379983270786, Tokens per sec: 279687.1494374946, Loss: 8.707337379455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1691 , TFLOPS: 19.86712529044823, Tokens per sec: 280644.1234370964, Loss: 8.700654983520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1692 , TFLOPS: 19.813563891426682, Tokens per sec: 279887.5121177077, Loss: 8.704798698425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1693 , TFLOPS: 19.868255437836478, Tokens per sec: 280660.0879623356, Loss: 8.706788063049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1694 , TFLOPS: 19.862400882620694, Tokens per sec: 280577.3861877896, Loss: 8.703948974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1695 , TFLOPS: 19.84502531010066, Tokens per sec: 280331.9378781921, Loss: 8.707097053527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1696 , TFLOPS: 19.86843613013222, Tokens per sec: 280662.6404318152, Loss: 8.7053861618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1697 , TFLOPS: 19.77621310612095, Tokens per sec: 279359.8928346676, Loss: 8.707524299621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1698 , TFLOPS: 19.87700613929958, Tokens per sec: 280783.7008608133, Loss: 8.700480461120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1699 , TFLOPS: 19.859637834050744, Tokens per sec: 280538.35520909535, Loss: 8.707518577575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1700 , TFLOPS: 19.5861320587104, Tokens per sec: 276674.797323734, Loss: 8.70944881439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1701 , TFLOPS: 19.787768718911423, Tokens per sec: 279523.12806749315, Loss: 8.706851959228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1702 , TFLOPS: 19.804736108912635, Tokens per sec: 279762.8103679916, Loss: 8.702170372009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1703 , TFLOPS: 19.82367762113389, Tokens per sec: 280030.37922942435, Loss: 8.703570365905762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1704 , TFLOPS: 19.265341202334863, Tokens per sec: 272143.28773802496, Loss: 8.702375411987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1705 , TFLOPS: 19.77745191568017, Tokens per sec: 279377.3923278137, Loss: 8.70547103881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1706 , TFLOPS: 19.843940654922417, Tokens per sec: 280316.61597845674, Loss: 8.704615592956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1707 , TFLOPS: 19.610725161426075, Tokens per sec: 277022.2008687026, Loss: 8.697916984558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1708 , TFLOPS: 19.856607414143937, Tokens per sec: 280495.547327937, Loss: 8.704255104064941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1709 , TFLOPS: 19.88808164165227, Tokens per sec: 280940.15402673214, Loss: 8.701545715332031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1710 , TFLOPS: 19.76520975045459, Tokens per sec: 279204.45881687663, Loss: 8.705286026000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1711 , TFLOPS: 19.829632030040646, Tokens per sec: 280114.4915427959, Loss: 8.707003593444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1712 , TFLOPS: 19.895129072914795, Tokens per sec: 281039.70643505687, Loss: 8.70156192779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1713 , TFLOPS: 19.85756185978657, Tokens per sec: 280509.02988048695, Loss: 8.704230308532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1714 , TFLOPS: 19.745182742218237, Tokens per sec: 278921.556178, Loss: 8.704440116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1715 , TFLOPS: 19.760925699723472, Tokens per sec: 279143.94207655214, Loss: 8.705791473388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1716 , TFLOPS: 19.8066600104192, Tokens per sec: 279789.98750831746, Loss: 8.705012321472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1717 , TFLOPS: 19.74464536681753, Tokens per sec: 278913.96518302296, Loss: 8.70537281036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1718 , TFLOPS: 18.987832847270568, Tokens per sec: 268223.1891875349, Loss: 8.705107688903809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1719 , TFLOPS: 19.874733913898716, Tokens per sec: 280751.60327766754, Loss: 8.701855659484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1720 , TFLOPS: 19.894085639140553, Tokens per sec: 281024.96683118085, Loss: 8.703158378601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1721 , TFLOPS: 19.71656936987162, Tokens per sec: 278517.36207928683, Loss: 8.700947761535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1722 , TFLOPS: 19.804501161503005, Tokens per sec: 279759.4914877376, Loss: 8.701923370361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1723 , TFLOPS: 19.74552061711108, Tokens per sec: 278926.3290176407, Loss: 8.700572967529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1724 , TFLOPS: 19.713454137712166, Tokens per sec: 278473.35613550217, Loss: 8.702491760253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1725 , TFLOPS: 19.86912111539335, Tokens per sec: 280672.31657193945, Loss: 8.704858779907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1726 , TFLOPS: 19.715223685689796, Tokens per sec: 278498.352869242, Loss: 8.702532768249512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1727 , TFLOPS: 19.773478168370012, Tokens per sec: 279321.25895097555, Loss: 8.70283317565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1728 , TFLOPS: 19.76411050172798, Tokens per sec: 279188.93076786044, Loss: 8.703951835632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1729 , TFLOPS: 19.837409795450768, Tokens per sec: 280224.3606921525, Loss: 8.702628135681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1730 , TFLOPS: 19.850541644170804, Tokens per sec: 280409.86192191095, Loss: 8.697624206542969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1731 , TFLOPS: 19.734086168759752, Tokens per sec: 278764.8053604618, Loss: 8.704299926757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1732 , TFLOPS: 19.760354451377747, Tokens per sec: 279135.8725904586, Loss: 8.703068733215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1733 , TFLOPS: 19.229569194151463, Tokens per sec: 271637.9703489489, Loss: 8.702776908874512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1734 , TFLOPS: 19.875253613445174, Tokens per sec: 280758.94458253856, Loss: 8.70449447631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1735 , TFLOPS: 19.850141582090817, Tokens per sec: 280404.2106225852, Loss: 8.703790664672852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1736 , TFLOPS: 19.81080961018972, Tokens per sec: 279848.6049868498, Loss: 8.702914237976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1737 , TFLOPS: 19.83346377054635, Tokens per sec: 280168.618923569, Loss: 8.703620910644531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1738 , TFLOPS: 19.871119713031515, Tokens per sec: 280700.54887399956, Loss: 8.701133728027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1739 , TFLOPS: 19.87403325377876, Tokens per sec: 280741.70571361046, Loss: 8.699740409851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1740 , TFLOPS: 19.822410968837513, Tokens per sec: 280012.4864282154, Loss: 8.702851295471191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1741 , TFLOPS: 19.739094280355165, Tokens per sec: 278835.55022506707, Loss: 8.702749252319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1742 , TFLOPS: 19.736231991888932, Tokens per sec: 278795.1173780454, Loss: 8.702447891235352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1743 , TFLOPS: 19.70527454460997, Tokens per sec: 278357.8107456847, Loss: 8.697516441345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1744 , TFLOPS: 19.731930090986335, Tokens per sec: 278734.34848519997, Loss: 8.70297622680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1745 , TFLOPS: 19.87754053008377, Tokens per sec: 280791.2496949305, Loss: 8.698189735412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1746 , TFLOPS: 19.676060993840846, Tokens per sec: 277945.13849299453, Loss: 8.705646514892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1747 , TFLOPS: 19.281456077798207, Tokens per sec: 272370.9273704625, Loss: 8.701973915100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1748 , TFLOPS: 19.84325447046899, Tokens per sec: 280306.922898475, Loss: 8.702948570251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1749 , TFLOPS: 19.746748558079926, Tokens per sec: 278943.6749804711, Loss: 8.702667236328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1750 , TFLOPS: 19.85788262117958, Tokens per sec: 280513.5609738715, Loss: 8.700729370117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1751 , TFLOPS: 19.844998571826384, Tokens per sec: 280331.5601718336, Loss: 8.697248458862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1752 , TFLOPS: 19.896743425426347, Tokens per sec: 281062.51086895965, Loss: 8.698542594909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1753 , TFLOPS: 19.909285924628694, Tokens per sec: 281239.68691445654, Loss: 8.700499534606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1754 , TFLOPS: 19.863236634813276, Tokens per sec: 280589.19207002316, Loss: 8.702326774597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1755 , TFLOPS: 19.909136611582586, Tokens per sec: 281237.577710013, Loss: 8.704298973083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1756 , TFLOPS: 19.90675886708453, Tokens per sec: 281203.98955819645, Loss: 8.700693130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1757 , TFLOPS: 19.701819399803778, Tokens per sec: 278309.00317683385, Loss: 8.700697898864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1758 , TFLOPS: 19.86976993215027, Tokens per sec: 280681.48179374955, Loss: 8.696305274963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1759 , TFLOPS: 19.845911421217266, Tokens per sec: 280344.45513340406, Loss: 8.703285217285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1760 , TFLOPS: 19.853317221283653, Tokens per sec: 280449.0698795032, Loss: 8.702880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1761 , TFLOPS: 19.22722428473895, Tokens per sec: 271604.8460273879, Loss: 8.700601577758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1762 , TFLOPS: 19.86035688028892, Tokens per sec: 280548.5124964871, Loss: 8.702922821044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1763 , TFLOPS: 19.911819874155213, Tokens per sec: 281275.48162724497, Loss: 8.702067375183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1764 , TFLOPS: 19.67875159980074, Tokens per sec: 277983.14614332595, Loss: 8.700820922851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1765 , TFLOPS: 19.827971842025025, Tokens per sec: 280091.0396339982, Loss: 8.698680877685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1766 , TFLOPS: 19.700622382057645, Tokens per sec: 278292.09403717774, Loss: 8.699737548828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1767 , TFLOPS: 19.858889240655568, Tokens per sec: 280527.7805369106, Loss: 8.698873519897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1768 , TFLOPS: 19.898241912165854, Tokens per sec: 281083.6785764837, Loss: 8.699688911437988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1769 , TFLOPS: 19.81476036885973, Tokens per sec: 279904.413624871, Loss: 8.69932746887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1770 , TFLOPS: 19.87270558088791, Tokens per sec: 280722.95093206974, Loss: 8.701306343078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1771 , TFLOPS: 19.877683235887815, Tokens per sec: 280793.2655651029, Loss: 8.69495677947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1772 , TFLOPS: 19.716582121610767, Tokens per sec: 278517.54221106746, Loss: 8.700309753417969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1773 , TFLOPS: 19.65851693529868, Tokens per sec: 277697.3101404215, Loss: 8.705403327941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1774 , TFLOPS: 19.686486149189992, Tokens per sec: 278092.4048207538, Loss: 8.697994232177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1775 , TFLOPS: 19.79605247154659, Tokens per sec: 279640.1448207012, Loss: 8.703007698059082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1776 , TFLOPS: 18.95991469104095, Tokens per sec: 267828.81575058843, Loss: 8.699337005615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1777 , TFLOPS: 19.834124952205418, Tokens per sec: 280177.9588126787, Loss: 8.70113468170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1778 , TFLOPS: 19.761823598927876, Tokens per sec: 279156.6258509519, Loss: 8.698674201965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1779 , TFLOPS: 19.849094155517435, Tokens per sec: 280389.41461620515, Loss: 8.702254295349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1780 , TFLOPS: 19.775802088639036, Tokens per sec: 279354.0867787213, Loss: 8.699856758117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1781 , TFLOPS: 19.7147191039859, Tokens per sec: 278491.225119861, Loss: 8.699167251586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1782 , TFLOPS: 19.78400002757866, Tokens per sec: 279469.89132286474, Loss: 8.699893951416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1783 , TFLOPS: 19.81377135922731, Tokens per sec: 279890.4428194674, Loss: 8.698821067810059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1784 , TFLOPS: 19.86325001154892, Tokens per sec: 280589.38103053934, Loss: 8.70043659210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1785 , TFLOPS: 19.75551041667924, Tokens per sec: 279067.44548527914, Loss: 8.696089744567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1786 , TFLOPS: 19.823220858872627, Tokens per sec: 280023.9269801655, Loss: 8.702054977416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1787 , TFLOPS: 19.769875479081044, Tokens per sec: 279270.3671605058, Loss: 8.698589324951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1788 , TFLOPS: 19.834572900720705, Tokens per sec: 280184.28655846877, Loss: 8.700568199157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1789 , TFLOPS: 19.738824115193356, Tokens per sec: 278831.7338568754, Loss: 8.699652671813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1790 , TFLOPS: 19.067171649921153, Tokens per sec: 269343.93355285726, Loss: 8.698455810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1791 , TFLOPS: 19.783622689098774, Tokens per sec: 279464.5610183854, Loss: 8.699148178100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1792 , TFLOPS: 19.693910109960314, Tokens per sec: 278197.2761059729, Loss: 8.700732231140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1793 , TFLOPS: 19.757238005242726, Tokens per sec: 279091.8494979872, Loss: 8.70202922821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1794 , TFLOPS: 19.62630801345671, Tokens per sec: 277242.3251083625, Loss: 8.698339462280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1795 , TFLOPS: 19.806354269381337, Tokens per sec: 279785.66859331005, Loss: 8.697664260864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1796 , TFLOPS: 19.67643062227047, Tokens per sec: 277950.35988487245, Loss: 8.6995267868042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1797 , TFLOPS: 19.683773136971716, Tokens per sec: 278054.08065836283, Loss: 8.703237533569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1798 , TFLOPS: 19.879618915569203, Tokens per sec: 280820.6090845853, Loss: 8.69748306274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1799 , TFLOPS: 19.746047920980494, Tokens per sec: 278933.7777416032, Loss: 8.69720458984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1800 , TFLOPS: 19.820756654900105, Tokens per sec: 279989.1174969751, Loss: 8.700799942016602 +------------------------------------------------------------------ + eval ppl=5994.22705078125, eval loss=8.698552131652832 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0001800, took 0.21s +[rank0]:[2024-08-03 20:52:23,062] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0014420419993257383, 'preprocessing_with_comm': 0.0006955919998290483, 'state_converting': 0.03647567999905732, : 0.038938392001000466}) +------------------------------------------------------------------ +iteration: 1801 , TFLOPS: 19.88101856299245, Tokens per sec: 280840.3805823976, Loss: 8.698272705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1802 , TFLOPS: 19.89070704024634, Tokens per sec: 280977.24055416544, Loss: 8.70086669921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1803 , TFLOPS: 19.879554370297935, Tokens per sec: 280819.697314472, Loss: 8.70246696472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1804 , TFLOPS: 19.878021260862006, Tokens per sec: 280798.0405248012, Loss: 8.698509216308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1805 , TFLOPS: 19.85295234017857, Tokens per sec: 280443.91554859735, Loss: 8.698020935058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1806 , TFLOPS: 19.425746802700434, Tokens per sec: 274409.1862236322, Loss: 8.701128005981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1807 , TFLOPS: 19.692728802907126, Tokens per sec: 278180.5888964443, Loss: 8.70217227935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1808 , TFLOPS: 19.78691828218568, Tokens per sec: 279511.1147507226, Loss: 8.70195198059082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1809 , TFLOPS: 19.79293281982564, Tokens per sec: 279596.0764459436, Loss: 8.702056884765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1810 , TFLOPS: 19.778243358738646, Tokens per sec: 279388.5722967397, Loss: 8.69982624053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1811 , TFLOPS: 19.626437025070178, Tokens per sec: 277244.1475336327, Loss: 8.697707176208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1812 , TFLOPS: 19.856880928495496, Tokens per sec: 280499.4110069705, Loss: 8.698838233947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1813 , TFLOPS: 19.80184158731595, Tokens per sec: 279721.9221939656, Loss: 8.697357177734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1814 , TFLOPS: 19.658521564363163, Tokens per sec: 277697.37553084537, Loss: 8.696556091308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1815 , TFLOPS: 19.804549573474155, Tokens per sec: 279760.1753579508, Loss: 8.699707984924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1816 , TFLOPS: 19.778385591604067, Tokens per sec: 279390.58148615464, Loss: 8.70068645477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1817 , TFLOPS: 19.831008660806248, Tokens per sec: 280133.9379060146, Loss: 8.698233604431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1818 , TFLOPS: 19.676170058602498, Tokens per sec: 277946.67914792, Loss: 8.701675415039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1819 , TFLOPS: 19.85292371516599, Tokens per sec: 280443.5111900681, Loss: 8.696953773498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1820 , TFLOPS: 19.798611255110256, Tokens per sec: 279676.2903404879, Loss: 8.697057723999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1821 , TFLOPS: 19.183214903293624, Tokens per sec: 270983.16704272525, Loss: 8.69933795928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1822 , TFLOPS: 19.743742184984836, Tokens per sec: 278901.2067859205, Loss: 8.69898509979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1823 , TFLOPS: 19.68936460289486, Tokens per sec: 278133.06601884094, Loss: 8.698503494262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1824 , TFLOPS: 19.851006407519563, Tokens per sec: 280416.4271949795, Loss: 8.697787284851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1825 , TFLOPS: 19.687057516951192, Tokens per sec: 278100.47599371703, Loss: 8.695176124572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1826 , TFLOPS: 19.83486268127984, Tokens per sec: 280188.38001486065, Loss: 8.69371223449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1827 , TFLOPS: 19.831813980704776, Tokens per sec: 280145.3138999596, Loss: 8.698863983154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1828 , TFLOPS: 19.784832684500582, Tokens per sec: 279481.65348113177, Loss: 8.697236061096191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1829 , TFLOPS: 19.808501765632553, Tokens per sec: 279816.00424550916, Loss: 8.695170402526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1830 , TFLOPS: 19.8131383281347, Tokens per sec: 279881.5005868345, Loss: 8.69598388671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1831 , TFLOPS: 19.799308064478733, Tokens per sec: 279686.13350861415, Loss: 8.701931953430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1832 , TFLOPS: 19.85597175155098, Tokens per sec: 280486.56792258297, Loss: 8.693799018859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1833 , TFLOPS: 19.675340646433458, Tokens per sec: 277934.96282521373, Loss: 8.69449234008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1834 , TFLOPS: 19.77094592391491, Tokens per sec: 279285.4883241213, Loss: 8.695050239562988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1835 , TFLOPS: 19.84485112968943, Tokens per sec: 280329.4773959575, Loss: 8.69749641418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1836 , TFLOPS: 19.27591504556177, Tokens per sec: 272292.6544390646, Loss: 8.696427345275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1837 , TFLOPS: 19.853201415141253, Tokens per sec: 280447.4339954562, Loss: 8.695769309997559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1838 , TFLOPS: 19.725945218537554, Tokens per sec: 278649.8058421292, Loss: 8.696852684020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1839 , TFLOPS: 19.871977749825643, Tokens per sec: 280712.66954976134, Loss: 8.693103790283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1840 , TFLOPS: 19.87115639578098, Tokens per sec: 280701.0670565709, Loss: 8.70011043548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1841 , TFLOPS: 19.865532142535365, Tokens per sec: 280621.6185405412, Loss: 8.696040153503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1842 , TFLOPS: 19.80013322570834, Tokens per sec: 279697.7897823121, Loss: 8.697322845458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1843 , TFLOPS: 19.81597617658865, Tokens per sec: 279921.58819287526, Loss: 8.697216987609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1844 , TFLOPS: 19.852353748777123, Tokens per sec: 280435.4598129688, Loss: 8.698528289794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1845 , TFLOPS: 19.85279660621216, Tokens per sec: 280441.7156418744, Loss: 8.695531845092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1846 , TFLOPS: 19.71311182836863, Tokens per sec: 278468.5206545628, Loss: 8.698986053466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1847 , TFLOPS: 19.814111963937442, Tokens per sec: 279895.25422065984, Loss: 8.693629264831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1848 , TFLOPS: 19.863852091919313, Tokens per sec: 280597.88605153846, Loss: 8.696802139282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1849 , TFLOPS: 19.8558824173616, Tokens per sec: 280485.3059828254, Loss: 8.69653034210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1850 , TFLOPS: 19.23835941379702, Tokens per sec: 271762.1414834831, Loss: 8.696514129638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1851 , TFLOPS: 19.846901960879567, Tokens per sec: 280358.4475520953, Loss: 8.700396537780762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1852 , TFLOPS: 19.787676432603202, Tokens per sec: 279521.8244259397, Loss: 8.697168350219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1853 , TFLOPS: 19.697707121524548, Tokens per sec: 278250.91290377564, Loss: 8.699190139770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1854 , TFLOPS: 19.71861826289925, Tokens per sec: 278546.30485683255, Loss: 8.698965072631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1855 , TFLOPS: 19.631667284704374, Tokens per sec: 277318.0304738633, Loss: 8.696406364440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1856 , TFLOPS: 19.707044388892264, Tokens per sec: 278382.81166504085, Loss: 8.697747230529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1857 , TFLOPS: 19.86647732563259, Tokens per sec: 280634.9702498565, Loss: 8.69670295715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1858 , TFLOPS: 19.749701701166813, Tokens per sec: 278985.3912449476, Loss: 8.694804191589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1859 , TFLOPS: 19.852540358312417, Tokens per sec: 280438.0958697054, Loss: 8.694144248962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1860 , TFLOPS: 19.868480208229908, Tokens per sec: 280663.2630814893, Loss: 8.695656776428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1861 , TFLOPS: 19.61121607402291, Tokens per sec: 277029.1355275125, Loss: 8.695683479309082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1862 , TFLOPS: 19.80914475871665, Tokens per sec: 279825.0872017912, Loss: 8.693285942077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1863 , TFLOPS: 19.726389224775076, Tokens per sec: 278656.07789908303, Loss: 8.697314262390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1864 , TFLOPS: 19.723008300039787, Tokens per sec: 278608.31876710645, Loss: 8.69079875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1865 , TFLOPS: 19.067112028527323, Tokens per sec: 269343.0913377121, Loss: 8.69572639465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1866 , TFLOPS: 19.828470078332373, Tokens per sec: 280098.0777479526, Loss: 8.698919296264648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1867 , TFLOPS: 19.568664729142455, Tokens per sec: 276428.05284894374, Loss: 8.695825576782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1868 , TFLOPS: 19.772247620829358, Tokens per sec: 279303.87616756593, Loss: 8.696243286132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1869 , TFLOPS: 19.739859895753384, Tokens per sec: 278846.3653510193, Loss: 8.696076393127441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1870 , TFLOPS: 19.82405949211246, Tokens per sec: 280035.7735602337, Loss: 8.693089485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1871 , TFLOPS: 19.72327633860801, Tokens per sec: 278612.1050949182, Loss: 8.694765090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1872 , TFLOPS: 19.701833535340093, Tokens per sec: 278309.2028562107, Loss: 8.69414234161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1873 , TFLOPS: 19.80991674321449, Tokens per sec: 279835.99229800096, Loss: 8.696625709533691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1874 , TFLOPS: 19.861522599424195, Tokens per sec: 280564.9795102151, Loss: 8.69372844696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1875 , TFLOPS: 19.86471585097509, Tokens per sec: 280610.0875602885, Loss: 8.697113037109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1876 , TFLOPS: 19.86188852207268, Tokens per sec: 280570.14855402126, Loss: 8.694844245910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1877 , TFLOPS: 19.779566714405423, Tokens per sec: 279407.26609293156, Loss: 8.697897911071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1878 , TFLOPS: 19.900806033255023, Tokens per sec: 281119.8994944542, Loss: 8.692753791809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1879 , TFLOPS: 19.21704837306948, Tokens per sec: 271461.1005297932, Loss: 8.697640419006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1880 , TFLOPS: 19.85700400462137, Tokens per sec: 280501.1495872108, Loss: 8.696773529052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1881 , TFLOPS: 19.84960070946617, Tokens per sec: 280396.5702256272, Loss: 8.694991111755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1882 , TFLOPS: 19.676483594034746, Tokens per sec: 277951.10816697846, Loss: 8.697362899780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1883 , TFLOPS: 19.734117407781884, Tokens per sec: 278765.2466446363, Loss: 8.698685646057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1884 , TFLOPS: 19.81805079163657, Tokens per sec: 279950.89432111883, Loss: 8.695123672485352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1885 , TFLOPS: 19.786381132420633, Tokens per sec: 279503.5269430875, Loss: 8.695015907287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1886 , TFLOPS: 19.794546887382317, Tokens per sec: 279618.8768545579, Loss: 8.696718215942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1887 , TFLOPS: 19.75768420064396, Tokens per sec: 279098.15247919003, Loss: 8.69636344909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1888 , TFLOPS: 19.809251089743352, Tokens per sec: 279826.5892398237, Loss: 8.6997652053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1889 , TFLOPS: 19.814839555792467, Tokens per sec: 279905.5322239134, Loss: 8.694173812866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1890 , TFLOPS: 19.856211470547752, Tokens per sec: 280489.95420654386, Loss: 8.689247131347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1891 , TFLOPS: 19.860490772583905, Tokens per sec: 280550.4038665375, Loss: 8.695334434509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1892 , TFLOPS: 19.869846895836737, Tokens per sec: 280682.56898709084, Loss: 8.691502571105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1893 , TFLOPS: 19.853198647524252, Tokens per sec: 280447.3948999436, Loss: 8.697013854980469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1894 , TFLOPS: 19.8297395082602, Tokens per sec: 280116.0097861389, Loss: 8.695804595947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1895 , TFLOPS: 19.228282183489068, Tokens per sec: 271619.78996432247, Loss: 8.693028450012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1896 , TFLOPS: 19.78654395201945, Tokens per sec: 279505.8269418503, Loss: 8.693744659423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1897 , TFLOPS: 19.853156923207383, Tokens per sec: 280446.80549990904, Loss: 8.694591522216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1898 , TFLOPS: 19.55039557073321, Tokens per sec: 276169.98169507826, Loss: 8.693171501159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1899 , TFLOPS: 19.804981099805595, Tokens per sec: 279766.27112305304, Loss: 8.69698715209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1900 , TFLOPS: 19.76512687193032, Tokens per sec: 279203.2880702056, Loss: 8.694698333740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1901 , TFLOPS: 19.827690889343216, Tokens per sec: 280087.0708806954, Loss: 8.697924613952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1902 , TFLOPS: 19.55690183689093, Tokens per sec: 276261.8895748539, Loss: 8.694701194763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1903 , TFLOPS: 19.473504185140794, Tokens per sec: 275083.8096800549, Loss: 8.694514274597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1904 , TFLOPS: 19.816082020048665, Tokens per sec: 279923.0833435135, Loss: 8.692152976989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1905 , TFLOPS: 19.79038088937528, Tokens per sec: 279560.027733619, Loss: 8.695632934570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1906 , TFLOPS: 19.57577730417412, Tokens per sec: 276528.5254817963, Loss: 8.698273658752441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1907 , TFLOPS: 19.88390094479723, Tokens per sec: 280881.0972690439, Loss: 8.696447372436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1908 , TFLOPS: 19.82801371090915, Tokens per sec: 280091.63107619795, Loss: 8.697558403015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1909 , TFLOPS: 19.247552212574575, Tokens per sec: 271891.99947336054, Loss: 8.695581436157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1910 , TFLOPS: 19.88019810067456, Tokens per sec: 280828.7906857895, Loss: 8.693516731262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1911 , TFLOPS: 19.863799620807438, Tokens per sec: 280597.1448416763, Loss: 8.696341514587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1912 , TFLOPS: 19.591055940813373, Tokens per sec: 276744.35235781636, Loss: 8.693455696105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1913 , TFLOPS: 19.769777561380394, Tokens per sec: 279268.9839695878, Loss: 8.692887306213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1914 , TFLOPS: 19.737730018570602, Tokens per sec: 278816.2785867667, Loss: 8.690482139587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1915 , TFLOPS: 19.689506180794563, Tokens per sec: 278135.0659561736, Loss: 8.692944526672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1916 , TFLOPS: 19.72487368151277, Tokens per sec: 278634.6692501636, Loss: 8.692232131958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1917 , TFLOPS: 19.87257653972943, Tokens per sec: 280721.1280894451, Loss: 8.689746856689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1918 , TFLOPS: 19.75050194139013, Tokens per sec: 278996.6954831164, Loss: 8.691606521606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1919 , TFLOPS: 19.758484133822655, Tokens per sec: 279109.4523800293, Loss: 8.691953659057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1920 , TFLOPS: 19.71232850493575, Tokens per sec: 278457.4553839224, Loss: 8.697673797607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1921 , TFLOPS: 19.811942620769987, Tokens per sec: 279864.6099576996, Loss: 8.694893836975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1922 , TFLOPS: 19.72487546263438, Tokens per sec: 278634.6944103872, Loss: 8.696669578552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1923 , TFLOPS: 19.825070450583272, Tokens per sec: 280050.05441615643, Loss: 8.693282127380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1924 , TFLOPS: 19.207245075513658, Tokens per sec: 271322.618600016, Loss: 8.693674087524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1925 , TFLOPS: 19.727289975745943, Tokens per sec: 278668.8019576954, Loss: 8.691473007202148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1926 , TFLOPS: 19.852687399214577, Tokens per sec: 280440.1729777163, Loss: 8.695188522338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1927 , TFLOPS: 19.71680762808302, Tokens per sec: 278520.7277281118, Loss: 8.693259239196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1928 , TFLOPS: 19.700968425928803, Tokens per sec: 278296.9822722632, Loss: 8.695069313049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1929 , TFLOPS: 19.85954351862932, Tokens per sec: 280537.0229041753, Loss: 8.689637184143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1930 , TFLOPS: 19.601821406069586, Tokens per sec: 276896.4259223635, Loss: 8.69517993927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1931 , TFLOPS: 19.849539586663287, Tokens per sec: 280395.70680149377, Loss: 8.69336986541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1932 , TFLOPS: 19.82416708577076, Tokens per sec: 280037.2934342708, Loss: 8.69334602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1933 , TFLOPS: 19.807784193722387, Tokens per sec: 279805.86778455734, Loss: 8.695820808410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1934 , TFLOPS: 19.69254231582601, Tokens per sec: 278177.9545695024, Loss: 8.695344924926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1935 , TFLOPS: 19.704998782329575, Tokens per sec: 278353.9153122827, Loss: 8.6970796585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1936 , TFLOPS: 19.799243135628515, Tokens per sec: 279685.21632004314, Loss: 8.695720672607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1937 , TFLOPS: 19.81084592876921, Tokens per sec: 279849.1180251358, Loss: 8.69577693939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1938 , TFLOPS: 19.79540924233436, Tokens per sec: 279631.05852886004, Loss: 8.692811012268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1939 , TFLOPS: 19.215372997181795, Tokens per sec: 271437.43407626497, Loss: 8.694710731506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1940 , TFLOPS: 19.805904482290085, Tokens per sec: 279779.31487569306, Loss: 8.694526672363281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1941 , TFLOPS: 19.67414896232708, Tokens per sec: 277918.1290288512, Loss: 8.692856788635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1942 , TFLOPS: 19.680237760186643, Tokens per sec: 278004.13972808927, Loss: 8.693826675415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1943 , TFLOPS: 19.864021377145026, Tokens per sec: 280600.27738410846, Loss: 8.694812774658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1944 , TFLOPS: 19.902879797343992, Tokens per sec: 281149.19360200403, Loss: 8.694262504577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1945 , TFLOPS: 19.615916368587175, Tokens per sec: 277095.5321515132, Loss: 8.689897537231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1946 , TFLOPS: 19.8399163355212, Tokens per sec: 280259.7682174307, Loss: 8.6961669921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1947 , TFLOPS: 19.814604037511906, Tokens per sec: 279902.20527950476, Loss: 8.691038131713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1948 , TFLOPS: 19.727100268394263, Tokens per sec: 278666.1221410312, Loss: 8.690327644348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1949 , TFLOPS: 19.87589911859926, Tokens per sec: 280768.06302446174, Loss: 8.695192337036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1950 , TFLOPS: 19.697919962966033, Tokens per sec: 278253.9195138867, Loss: 8.692628860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1951 , TFLOPS: 19.466331474181022, Tokens per sec: 274982.4875637173, Loss: 8.694332122802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1952 , TFLOPS: 19.805300914073175, Tokens per sec: 279770.7888372908, Loss: 8.693306922912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1953 , TFLOPS: 19.20768428614771, Tokens per sec: 271328.8229140069, Loss: 8.6915283203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1954 , TFLOPS: 19.802058561201868, Tokens per sec: 279724.98717921803, Loss: 8.694990158081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1955 , TFLOPS: 19.563593902303676, Tokens per sec: 276356.4220653017, Loss: 8.692310333251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1956 , TFLOPS: 19.849135293202597, Tokens per sec: 280389.99572944746, Loss: 8.69251823425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1957 , TFLOPS: 19.684925740686307, Tokens per sec: 278070.3624029244, Loss: 8.693008422851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1958 , TFLOPS: 19.873042117981782, Tokens per sec: 280727.7048738874, Loss: 8.68893051147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1959 , TFLOPS: 19.856086755431587, Tokens per sec: 280488.19247383496, Loss: 8.692852973937988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1960 , TFLOPS: 19.796910964679714, Tokens per sec: 279652.2719427332, Loss: 8.692733764648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1961 , TFLOPS: 19.860417037914377, Tokens per sec: 280549.36228647025, Loss: 8.69143295288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1962 , TFLOPS: 19.87502282286744, Tokens per sec: 280755.68442192447, Loss: 8.693312644958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1963 , TFLOPS: 19.754985811042765, Tokens per sec: 279060.03487670614, Loss: 8.696578979492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1964 , TFLOPS: 19.788167557918545, Tokens per sec: 279528.7620896232, Loss: 8.691350936889648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1965 , TFLOPS: 19.832101636435226, Tokens per sec: 280149.3773409026, Loss: 8.692713737487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1966 , TFLOPS: 19.754177625257572, Tokens per sec: 279048.6183990858, Loss: 8.69534969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1967 , TFLOPS: 19.88378380858847, Tokens per sec: 280879.44259640505, Loss: 8.692963600158691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1968 , TFLOPS: 19.210070291963373, Tokens per sec: 271362.5277656582, Loss: 8.690681457519531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1969 , TFLOPS: 19.811012163785016, Tokens per sec: 279851.46627027, Loss: 8.692665100097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1970 , TFLOPS: 19.81119528828791, Tokens per sec: 279854.0530972438, Loss: 8.68932056427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1971 , TFLOPS: 19.799081672119115, Tokens per sec: 279682.93547747534, Loss: 8.695601463317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1972 , TFLOPS: 19.697255471393877, Tokens per sec: 278244.53286875953, Loss: 8.69459056854248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1973 , TFLOPS: 19.69001832684971, Tokens per sec: 278142.30056000367, Loss: 8.693784713745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1974 , TFLOPS: 19.838952815275842, Tokens per sec: 280246.15747653495, Loss: 8.687765121459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1975 , TFLOPS: 19.810418273616577, Tokens per sec: 279843.0769445219, Loss: 8.694518089294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1976 , TFLOPS: 19.84708905095812, Tokens per sec: 280361.0903970128, Loss: 8.690081596374512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1977 , TFLOPS: 19.719415343622796, Tokens per sec: 278557.5644637308, Loss: 8.691390991210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1978 , TFLOPS: 19.805318351061533, Tokens per sec: 279771.03515316395, Loss: 8.694354057312012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1979 , TFLOPS: 19.685885995624453, Tokens per sec: 278083.92701790784, Loss: 8.689273834228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1980 , TFLOPS: 19.748430523586244, Tokens per sec: 278967.4345193209, Loss: 8.694735527038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1981 , TFLOPS: 19.77688035095219, Tokens per sec: 279369.3183724866, Loss: 8.69120979309082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1982 , TFLOPS: 19.17528467134097, Tokens per sec: 270871.14414245816, Loss: 8.692671775817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1983 , TFLOPS: 19.84520541067135, Tokens per sec: 280334.48198893043, Loss: 8.69290828704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1984 , TFLOPS: 19.80006075492006, Tokens per sec: 279696.76605590206, Loss: 8.693954467773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1985 , TFLOPS: 19.706321062692382, Tokens per sec: 278372.5939186705, Loss: 8.691518783569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1986 , TFLOPS: 19.782792060320016, Tokens per sec: 279452.8275097822, Loss: 8.691569328308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1987 , TFLOPS: 19.850699268747036, Tokens per sec: 280412.0885354941, Loss: 8.693693161010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1988 , TFLOPS: 19.7435177659549, Tokens per sec: 278898.0366301471, Loss: 8.691640853881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1989 , TFLOPS: 19.70918964300005, Tokens per sec: 278413.1156445964, Loss: 8.690731048583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1990 , TFLOPS: 19.8906479245909, Tokens per sec: 280976.4054831091, Loss: 8.690079689025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1991 , TFLOPS: 19.862403713900864, Tokens per sec: 280577.42618261167, Loss: 8.696847915649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1992 , TFLOPS: 19.835888670963527, Tokens per sec: 280202.8732025375, Loss: 8.692533493041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1993 , TFLOPS: 19.602524073047917, Tokens per sec: 276906.35183541424, Loss: 8.692755699157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1994 , TFLOPS: 19.85810329529421, Tokens per sec: 280516.67822876165, Loss: 8.692703247070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1995 , TFLOPS: 19.862780926695752, Tokens per sec: 280582.75471165765, Loss: 8.68855094909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1996 , TFLOPS: 19.456695222052318, Tokens per sec: 274846.3653269899, Loss: 8.688109397888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1997 , TFLOPS: 19.815946793099968, Tokens per sec: 279921.17312007013, Loss: 8.691248893737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1998 , TFLOPS: 19.764259233939157, Tokens per sec: 279191.03176739725, Loss: 8.693952560424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1999 , TFLOPS: 19.842828315229244, Tokens per sec: 280300.9030057145, Loss: 8.689993858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2000 , TFLOPS: 19.80916487676367, Tokens per sec: 279825.37139044906, Loss: 8.693239212036133 +------------------------------------------------------------------ + eval ppl=5977.6376953125, eval loss=8.695780754089355 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002000, took 0.21s +[rank0]:[2024-08-03 21:02:12,806] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0009974399999919115, 'preprocessing_with_comm': 0.0007948790007503703, 'state_converting': 0.028250666999156238, : 0.03037819900055183}) +------------------------------------------------------------------ +iteration: 2001 , TFLOPS: 19.853554834319088, Tokens per sec: 280452.4264145364, Loss: 8.692354202270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2002 , TFLOPS: 19.82399589552553, Tokens per sec: 280034.8751912884, Loss: 8.689125061035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2003 , TFLOPS: 19.71764145676142, Tokens per sec: 278532.5064386764, Loss: 8.694012641906738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2004 , TFLOPS: 19.846468463482935, Tokens per sec: 280352.3239436159, Loss: 8.694311141967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2005 , TFLOPS: 19.649767733648392, Tokens per sec: 277573.7184283835, Loss: 8.692756652832031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2006 , TFLOPS: 19.72401227211566, Tokens per sec: 278622.50093282346, Loss: 8.693683624267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2007 , TFLOPS: 19.846458668186102, Tokens per sec: 280352.1855747048, Loss: 8.689613342285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2008 , TFLOPS: 19.81302396310644, Tokens per sec: 279879.88506004645, Loss: 8.689842224121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2009 , TFLOPS: 19.812721720941756, Tokens per sec: 279875.61557031743, Loss: 8.69131088256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2010 , TFLOPS: 19.757342725018773, Tokens per sec: 279093.3287754028, Loss: 8.691779136657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2011 , TFLOPS: 19.234844683300818, Tokens per sec: 271712.4922038413, Loss: 8.689220428466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2012 , TFLOPS: 19.807372018201175, Tokens per sec: 279800.04537008307, Loss: 8.694741249084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2013 , TFLOPS: 19.792147933459873, Tokens per sec: 279584.9890972257, Loss: 8.689947128295898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2014 , TFLOPS: 19.825290876303924, Tokens per sec: 280053.1681622194, Loss: 8.69321346282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2015 , TFLOPS: 19.782767038594244, Tokens per sec: 279452.47405148397, Loss: 8.69324779510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2016 , TFLOPS: 19.621711935520864, Tokens per sec: 277177.4006542859, Loss: 8.692547798156738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2017 , TFLOPS: 19.793670393851542, Tokens per sec: 279606.4954579015, Loss: 8.691303253173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2018 , TFLOPS: 19.766759270455633, Tokens per sec: 279226.3474231084, Loss: 8.694977760314941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2019 , TFLOPS: 19.84772066019352, Tokens per sec: 280370.0125444112, Loss: 8.691657066345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2020 , TFLOPS: 19.81940750118453, Tokens per sec: 279970.05927610246, Loss: 8.692082405090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2021 , TFLOPS: 19.832377826822068, Tokens per sec: 280153.2788217602, Loss: 8.690604209899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2022 , TFLOPS: 19.835572892401633, Tokens per sec: 280198.412496904, Loss: 8.694635391235352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2023 , TFLOPS: 19.805083950121066, Tokens per sec: 279767.72399236396, Loss: 8.694324493408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2024 , TFLOPS: 19.61656984027346, Tokens per sec: 277104.76312911604, Loss: 8.693821907043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2025 , TFLOPS: 19.77334784892294, Tokens per sec: 279319.4180511746, Loss: 8.69192886352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2026 , TFLOPS: 19.85533765625051, Tokens per sec: 280477.61065689026, Loss: 8.693726539611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2027 , TFLOPS: 19.10301409834028, Tokens per sec: 269850.24598464434, Loss: 8.686867713928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2028 , TFLOPS: 19.633187411673394, Tokens per sec: 277339.5038724801, Loss: 8.691667556762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2029 , TFLOPS: 19.63985522833672, Tokens per sec: 277433.6938237369, Loss: 8.689826965332031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2030 , TFLOPS: 19.6259555189799, Tokens per sec: 277237.34575166076, Loss: 8.689492225646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2031 , TFLOPS: 19.83576308016626, Tokens per sec: 280201.0990999083, Loss: 8.692538261413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2032 , TFLOPS: 19.70302609698412, Tokens per sec: 278326.04904871923, Loss: 8.688846588134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2033 , TFLOPS: 19.887568758582482, Tokens per sec: 280932.909011792, Loss: 8.688591003417969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2034 , TFLOPS: 19.916166822594413, Tokens per sec: 281336.8868641129, Loss: 8.69205379486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2035 , TFLOPS: 19.896170504187918, Tokens per sec: 281054.4177514901, Loss: 8.692331314086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2036 , TFLOPS: 19.863289992479988, Tokens per sec: 280589.9458034085, Loss: 8.691171646118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2037 , TFLOPS: 19.83607205249912, Tokens per sec: 280205.46366036957, Loss: 8.687390327453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2038 , TFLOPS: 19.77572061543707, Tokens per sec: 279352.93588371173, Loss: 8.684675216674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2039 , TFLOPS: 19.650313001910376, Tokens per sec: 277581.4209183605, Loss: 8.695838928222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2040 , TFLOPS: 19.66451748357201, Tokens per sec: 277782.07422106684, Loss: 8.694657325744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2041 , TFLOPS: 19.843186210398475, Tokens per sec: 280305.9586528997, Loss: 8.6898193359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2042 , TFLOPS: 19.261473086558137, Tokens per sec: 272088.6465181382, Loss: 8.691333770751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2043 , TFLOPS: 19.67617356330529, Tokens per sec: 277946.72865554766, Loss: 8.689793586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2044 , TFLOPS: 19.712235088629356, Tokens per sec: 278456.13577995147, Loss: 8.691622734069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2045 , TFLOPS: 19.839699865714326, Tokens per sec: 280256.71035282715, Loss: 8.687508583068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2046 , TFLOPS: 19.888706059717084, Tokens per sec: 280948.9745912524, Loss: 8.691579818725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2047 , TFLOPS: 19.67063081273691, Tokens per sec: 277868.43145089666, Loss: 8.693575859069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2048 , TFLOPS: 19.6347569651094, Tokens per sec: 277361.6754721352, Loss: 8.68898868560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2049 , TFLOPS: 19.80403480680764, Tokens per sec: 279752.90373521793, Loss: 8.690260887145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2050 , TFLOPS: 19.792003190196134, Tokens per sec: 279582.9444457836, Loss: 8.690999031066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2051 , TFLOPS: 19.847649799949288, Tokens per sec: 280369.01156863646, Loss: 8.688632011413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2052 , TFLOPS: 19.783688162194547, Tokens per sec: 279465.48589499964, Loss: 8.688058853149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2053 , TFLOPS: 19.860979931971322, Tokens per sec: 280557.3137594131, Loss: 8.690057754516602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2054 , TFLOPS: 19.852148192040225, Tokens per sec: 280432.5561070023, Loss: 8.689940452575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2055 , TFLOPS: 19.89227916760857, Tokens per sec: 280999.4485132463, Loss: 8.691441535949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2056 , TFLOPS: 19.586114655634937, Tokens per sec: 276674.5514869162, Loss: 8.688650131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2057 , TFLOPS: 19.211014406801777, Tokens per sec: 271375.86438468954, Loss: 8.689017295837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2058 , TFLOPS: 19.85203159919595, Tokens per sec: 280430.9091099606, Loss: 8.691621780395508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2059 , TFLOPS: 19.89529884398312, Tokens per sec: 281042.1046306669, Loss: 8.693937301635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2060 , TFLOPS: 19.732655834533155, Tokens per sec: 278744.6003791484, Loss: 8.690165519714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2061 , TFLOPS: 19.85514216046313, Tokens per sec: 280474.8490724596, Loss: 8.684452056884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2062 , TFLOPS: 19.744283724555725, Tokens per sec: 278908.8566041985, Loss: 8.692188262939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2063 , TFLOPS: 19.90181892468704, Tokens per sec: 281134.2076554951, Loss: 8.686214447021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2064 , TFLOPS: 19.899600464289296, Tokens per sec: 281102.86955979164, Loss: 8.68944263458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2065 , TFLOPS: 19.676507601333586, Tokens per sec: 277951.44729542517, Loss: 8.6863374710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2066 , TFLOPS: 19.794200009588202, Tokens per sec: 279613.97683943005, Loss: 8.685724258422852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2067 , TFLOPS: 19.67356061824107, Tokens per sec: 277909.81804737373, Loss: 8.690657615661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2068 , TFLOPS: 19.59372687560873, Tokens per sec: 276782.0821321769, Loss: 8.69155502319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2069 , TFLOPS: 19.81099140705911, Tokens per sec: 279851.1730595985, Loss: 8.690125465393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2070 , TFLOPS: 19.70308408260897, Tokens per sec: 278326.86815690016, Loss: 8.68733024597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2071 , TFLOPS: 19.07462686034577, Tokens per sec: 269449.24627244135, Loss: 8.690855979919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2072 , TFLOPS: 19.726569234440777, Tokens per sec: 278658.6207256924, Loss: 8.691165924072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2073 , TFLOPS: 19.824275025546378, Tokens per sec: 280038.8181975805, Loss: 8.687185287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2074 , TFLOPS: 19.67916581776087, Tokens per sec: 277988.9974094067, Loss: 8.690019607543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2075 , TFLOPS: 19.57358175694953, Tokens per sec: 276497.51105885877, Loss: 8.695158004760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2076 , TFLOPS: 19.550636111865465, Tokens per sec: 276173.379592571, Loss: 8.688858032226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2077 , TFLOPS: 19.789545339013227, Tokens per sec: 279548.22470245475, Loss: 8.687019348144531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2078 , TFLOPS: 19.57590515109416, Tokens per sec: 276530.3314545419, Loss: 8.692730903625488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2079 , TFLOPS: 19.87384972027616, Tokens per sec: 280739.11310908385, Loss: 8.684399604797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2080 , TFLOPS: 19.83902652611286, Tokens per sec: 280247.1987199429, Loss: 8.687742233276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2081 , TFLOPS: 19.585612805488317, Tokens per sec: 276667.4623236661, Loss: 8.688887596130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2082 , TFLOPS: 19.882425251672334, Tokens per sec: 280860.25154539524, Loss: 8.690351486206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2083 , TFLOPS: 19.825991540260034, Tokens per sec: 280063.0657804662, Loss: 8.691882133483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2084 , TFLOPS: 19.87637086302646, Tokens per sec: 280774.7269126349, Loss: 8.68896484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2085 , TFLOPS: 19.83389686927345, Tokens per sec: 280174.7369004202, Loss: 8.687332153320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2086 , TFLOPS: 19.21298589000184, Tokens per sec: 271403.71366668, Loss: 8.691422462463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2087 , TFLOPS: 19.804890056960758, Tokens per sec: 279764.98504673294, Loss: 8.68887996673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2088 , TFLOPS: 19.87657225053873, Tokens per sec: 280777.571723901, Loss: 8.690390586853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2089 , TFLOPS: 19.887066676678586, Tokens per sec: 280925.8165747256, Loss: 8.69125747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2090 , TFLOPS: 19.743161310319188, Tokens per sec: 278893.00131789304, Loss: 8.688982009887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2091 , TFLOPS: 19.757866985556138, Tokens per sec: 279100.73450908647, Loss: 8.690613746643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2092 , TFLOPS: 19.727964399032373, Tokens per sec: 278678.3288987741, Loss: 8.686932563781738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2093 , TFLOPS: 19.826074294097033, Tokens per sec: 280064.2347657978, Loss: 8.686468124389648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2094 , TFLOPS: 19.785662703136378, Tokens per sec: 279493.3783708213, Loss: 8.694449424743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2095 , TFLOPS: 19.865852502675942, Tokens per sec: 280626.1439658114, Loss: 8.688295364379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2096 , TFLOPS: 19.880509730843404, Tokens per sec: 280833.19279099064, Loss: 8.688691139221191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2097 , TFLOPS: 19.78877499531367, Tokens per sec: 279537.3427842534, Loss: 8.691839218139648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2098 , TFLOPS: 19.80798260695635, Tokens per sec: 279808.67058100173, Loss: 8.689037322998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2099 , TFLOPS: 19.733425876344885, Tokens per sec: 278755.4780328654, Loss: 8.691818237304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2100 , TFLOPS: 19.12280474204363, Tokens per sec: 270129.80972490285, Loss: 8.688196182250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2101 , TFLOPS: 19.85329912988539, Tokens per sec: 280448.8143193991, Loss: 8.689512252807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2102 , TFLOPS: 19.800552090345825, Tokens per sec: 279703.70668761706, Loss: 8.687634468078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2103 , TFLOPS: 19.720040626327062, Tokens per sec: 278566.3972421964, Loss: 8.689973831176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2104 , TFLOPS: 19.905144416429422, Tokens per sec: 281181.18373791233, Loss: 8.690032005310059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2105 , TFLOPS: 19.769026337528835, Tokens per sec: 279258.3721394289, Loss: 8.690279960632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2106 , TFLOPS: 19.767091448703265, Tokens per sec: 279231.0397916201, Loss: 8.686132431030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2107 , TFLOPS: 19.858895912519387, Tokens per sec: 280527.87478403223, Loss: 8.685182571411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2108 , TFLOPS: 19.873964129224245, Tokens per sec: 280740.7292562858, Loss: 8.685837745666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2109 , TFLOPS: 19.682262447848693, Tokens per sec: 278032.7405792866, Loss: 8.690402030944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2110 , TFLOPS: 19.85878090325677, Tokens per sec: 280526.2501567536, Loss: 8.685773849487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2111 , TFLOPS: 19.71360770658668, Tokens per sec: 278475.5254580159, Loss: 8.686433792114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2112 , TFLOPS: 19.557414822667123, Tokens per sec: 276269.1360406304, Loss: 8.691926002502441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2113 , TFLOPS: 19.755696634007432, Tokens per sec: 279070.07600167627, Loss: 8.685656547546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2114 , TFLOPS: 19.84408944574099, Tokens per sec: 280318.7178058849, Loss: 8.689971923828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2115 , TFLOPS: 19.447616628573893, Tokens per sec: 274718.1206075579, Loss: 8.688166618347168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2116 , TFLOPS: 19.81260990084358, Tokens per sec: 279874.0359933541, Loss: 8.690343856811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2117 , TFLOPS: 19.82969669142041, Tokens per sec: 280115.4049530643, Loss: 8.68845272064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2118 , TFLOPS: 19.88666332376597, Tokens per sec: 280920.11878891406, Loss: 8.688387870788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2119 , TFLOPS: 19.750832009176662, Tokens per sec: 279001.3580391365, Loss: 8.691227912902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2120 , TFLOPS: 19.91971291541987, Tokens per sec: 281386.97916977253, Loss: 8.686529159545898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2121 , TFLOPS: 19.837320634964517, Tokens per sec: 280223.1012061352, Loss: 8.684748649597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2122 , TFLOPS: 19.857284295690906, Tokens per sec: 280505.10899454163, Loss: 8.690494537353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2123 , TFLOPS: 19.854733161037746, Tokens per sec: 280469.0715236927, Loss: 8.689184188842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2124 , TFLOPS: 19.84039266701391, Tokens per sec: 280266.49690274795, Loss: 8.690917015075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2125 , TFLOPS: 19.83764488768372, Tokens per sec: 280227.68161818886, Loss: 8.68639850616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2126 , TFLOPS: 19.621861063455682, Tokens per sec: 277179.507243837, Loss: 8.688896179199219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2127 , TFLOPS: 19.759871055815292, Tokens per sec: 279129.0441176969, Loss: 8.688128471374512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2128 , TFLOPS: 19.894453437007325, Tokens per sec: 281030.1623644255, Loss: 8.688779830932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2129 , TFLOPS: 19.424026432685388, Tokens per sec: 274384.88418054336, Loss: 8.687332153320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2130 , TFLOPS: 19.8988921619989, Tokens per sec: 281092.86404200736, Loss: 8.687773704528809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2131 , TFLOPS: 19.627972374225216, Tokens per sec: 277265.83596170094, Loss: 8.688263893127441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2132 , TFLOPS: 19.73020962522714, Tokens per sec: 278710.0450896239, Loss: 8.689081192016602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2133 , TFLOPS: 19.877261020044756, Tokens per sec: 280787.30132048053, Loss: 8.689522743225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2134 , TFLOPS: 19.839753002167004, Tokens per sec: 280257.4609613307, Loss: 8.688547134399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2135 , TFLOPS: 19.83705271812108, Tokens per sec: 280219.31659781456, Loss: 8.683469772338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2136 , TFLOPS: 19.78518889485758, Tokens per sec: 279486.68532856496, Loss: 8.685330390930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2137 , TFLOPS: 19.802080459962188, Tokens per sec: 279725.29652233154, Loss: 8.684415817260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2138 , TFLOPS: 19.820968823756594, Tokens per sec: 279992.11460611294, Loss: 8.691957473754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2139 , TFLOPS: 19.82445060352005, Tokens per sec: 280041.2984218603, Loss: 8.691351890563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2140 , TFLOPS: 19.847538520812325, Tokens per sec: 280367.43963332154, Loss: 8.68468189239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2141 , TFLOPS: 19.862568384573486, Tokens per sec: 280579.7523297466, Loss: 8.688368797302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2142 , TFLOPS: 19.825084614721728, Tokens per sec: 280050.2544995688, Loss: 8.687727928161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2143 , TFLOPS: 19.82478205859513, Tokens per sec: 280045.98057479615, Loss: 8.687378883361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2144 , TFLOPS: 19.42917062367922, Tokens per sec: 274457.55131034454, Loss: 8.689624786376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2145 , TFLOPS: 19.82582685992022, Tokens per sec: 280060.739496772, Loss: 8.684995651245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2146 , TFLOPS: 19.741149474148983, Tokens per sec: 278864.58200758864, Loss: 8.687369346618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2147 , TFLOPS: 19.75081063986201, Tokens per sec: 279001.0561750021, Loss: 8.689257621765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2148 , TFLOPS: 19.80437818186186, Tokens per sec: 279757.754270446, Loss: 8.683897972106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2149 , TFLOPS: 19.873453224702807, Tokens per sec: 280733.5121904304, Loss: 8.687274932861328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2150 , TFLOPS: 19.827738433684743, Tokens per sec: 280087.742494724, Loss: 8.682555198669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2151 , TFLOPS: 19.817725502866715, Tokens per sec: 279946.29927375563, Loss: 8.689817428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2152 , TFLOPS: 19.900088696476207, Tokens per sec: 281109.766354981, Loss: 8.686847686767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2153 , TFLOPS: 19.87285433439308, Tokens per sec: 280725.05223240674, Loss: 8.688624382019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2154 , TFLOPS: 19.848149215516344, Tokens per sec: 280376.0663408795, Loss: 8.688310623168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2155 , TFLOPS: 19.80277503668321, Tokens per sec: 279735.10815195495, Loss: 8.687207221984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2156 , TFLOPS: 19.81921176586727, Tokens per sec: 279967.2943080593, Loss: 8.680981636047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2157 , TFLOPS: 19.792351069693744, Tokens per sec: 279587.858611031, Loss: 8.68645191192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2158 , TFLOPS: 19.691558481756772, Tokens per sec: 278164.0568743939, Loss: 8.687009811401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2159 , TFLOPS: 19.517242863076465, Tokens per sec: 275701.6646917941, Loss: 8.686603546142578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2160 , TFLOPS: 19.80700678491761, Tokens per sec: 279794.88606428436, Loss: 8.6884183883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2161 , TFLOPS: 19.756110658373128, Tokens per sec: 279075.9245330312, Loss: 8.688179969787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2162 , TFLOPS: 19.55573921876651, Tokens per sec: 276245.46636617876, Loss: 8.683503150939941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2163 , TFLOPS: 19.751044717913132, Tokens per sec: 279004.3627746491, Loss: 8.68563175201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2164 , TFLOPS: 19.803600696154415, Tokens per sec: 279746.77146384155, Loss: 8.687631607055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2165 , TFLOPS: 19.802501229153528, Tokens per sec: 279731.24033147085, Loss: 8.68640422821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2166 , TFLOPS: 19.72310394070987, Tokens per sec: 278609.66979256325, Loss: 8.682390213012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2167 , TFLOPS: 19.902096514987473, Tokens per sec: 281138.1289116104, Loss: 8.689544677734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2168 , TFLOPS: 19.8992572549206, Tokens per sec: 281098.0213650465, Loss: 8.691988945007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2169 , TFLOPS: 19.77796424438242, Tokens per sec: 279384.6295117269, Loss: 8.686480522155762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2170 , TFLOPS: 19.88267353593284, Tokens per sec: 280863.75882275094, Loss: 8.68542766571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2171 , TFLOPS: 19.89116387408325, Tokens per sec: 280983.6938145059, Loss: 8.686362266540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2172 , TFLOPS: 19.822764219094076, Tokens per sec: 280017.4764611043, Loss: 8.688546180725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2173 , TFLOPS: 19.789830677941822, Tokens per sec: 279552.25541612506, Loss: 8.687106132507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2174 , TFLOPS: 19.52158243713489, Tokens per sec: 275762.96575774666, Loss: 8.689233779907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2175 , TFLOPS: 19.754393931966764, Tokens per sec: 279051.6739597624, Loss: 8.686824798583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2176 , TFLOPS: 19.79187170287844, Tokens per sec: 279581.087048577, Loss: 8.6864652633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2177 , TFLOPS: 19.711000430143038, Tokens per sec: 278438.69492509344, Loss: 8.690191268920898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2178 , TFLOPS: 19.856769864761322, Tokens per sec: 280497.84211444744, Loss: 8.682809829711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2179 , TFLOPS: 19.86090729344586, Tokens per sec: 280556.2876635382, Loss: 8.685484886169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2180 , TFLOPS: 19.763172056055552, Tokens per sec: 279175.6742317848, Loss: 8.690875053405762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2181 , TFLOPS: 19.820935923086083, Tokens per sec: 279991.64984940045, Loss: 8.686768531799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2182 , TFLOPS: 19.76645174084053, Tokens per sec: 279222.0032425589, Loss: 8.684333801269531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2183 , TFLOPS: 19.854693675884594, Tokens per sec: 280468.51375421, Loss: 8.682483673095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2184 , TFLOPS: 19.731417631762785, Tokens per sec: 278727.1094575355, Loss: 8.687128067016602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2185 , TFLOPS: 19.740343821671317, Tokens per sec: 278853.20131561207, Loss: 8.689176559448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2186 , TFLOPS: 19.914814996840423, Tokens per sec: 281317.79089787614, Loss: 8.69074821472168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2187 , TFLOPS: 19.826471411237566, Tokens per sec: 280069.8444647441, Loss: 8.687827110290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2188 , TFLOPS: 19.70588790376869, Tokens per sec: 278366.4750914788, Loss: 8.684930801391602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2189 , TFLOPS: 19.231411761480175, Tokens per sec: 271663.99855812767, Loss: 8.689774513244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2190 , TFLOPS: 19.85375699440787, Tokens per sec: 280455.28213926166, Loss: 8.685641288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2191 , TFLOPS: 19.7503492875011, Tokens per sec: 278994.53908573877, Loss: 8.68441390991211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2192 , TFLOPS: 19.68489287661787, Tokens per sec: 278069.8981632549, Loss: 8.68713092803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2193 , TFLOPS: 19.831346264368886, Tokens per sec: 280138.7069128296, Loss: 8.68732738494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2194 , TFLOPS: 19.820533355137762, Tokens per sec: 279985.96315203846, Loss: 8.686058044433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2195 , TFLOPS: 19.753549927411896, Tokens per sec: 279039.7515041974, Loss: 8.68547534942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2196 , TFLOPS: 19.75606834700744, Tokens per sec: 279075.32684031205, Loss: 8.687284469604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2197 , TFLOPS: 19.853782046776107, Tokens per sec: 280455.63603041705, Loss: 8.685344696044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2198 , TFLOPS: 19.912030598881167, Tokens per sec: 281278.458336514, Loss: 8.686978340148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2199 , TFLOPS: 19.841874167382457, Tokens per sec: 280287.42465984536, Loss: 8.68793773651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2200 , TFLOPS: 19.73863442732994, Tokens per sec: 278829.0543155036, Loss: 8.688258171081543 +------------------------------------------------------------------ +[rank0]:[2024-08-03 21:12:01,952] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00099673299882852, 'preprocessing_with_comm': 0.0007428589997289237, 'state_converting': 0.029694102000576095, : 0.03175305600052525}) + eval ppl=5886.70361328125, eval loss=8.680451393127441 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002200, took 0.18s +Deleting checkpoint /work/llm_recipes/models/tiny-mistral-sample2/iter_0000200 +------------------------------------------------------------------ +iteration: 2201 , TFLOPS: 19.82003394912199, Tokens per sec: 279978.90851471736, Loss: 8.686727523803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2202 , TFLOPS: 19.627519441021345, Tokens per sec: 277259.43780192104, Loss: 8.688146591186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2203 , TFLOPS: 19.850546208145943, Tokens per sec: 280409.92639287916, Loss: 8.687705993652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2204 , TFLOPS: 19.42471133883513, Tokens per sec: 274394.5592031358, Loss: 8.685403823852539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2205 , TFLOPS: 19.709828090084372, Tokens per sec: 278422.13438383123, Loss: 8.688652992248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2206 , TFLOPS: 19.89645671200818, Tokens per sec: 281058.46073917276, Loss: 8.688039779663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2207 , TFLOPS: 19.909305557609095, Tokens per sec: 281239.9642510362, Loss: 8.68657398223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2208 , TFLOPS: 19.697767266085744, Tokens per sec: 278251.76250921225, Loss: 8.684927940368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2209 , TFLOPS: 19.89754599846615, Tokens per sec: 281073.8480606246, Loss: 8.68969440460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2210 , TFLOPS: 19.823791290835317, Tokens per sec: 280031.9849339866, Loss: 8.687361717224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2211 , TFLOPS: 19.85671901892832, Tokens per sec: 280497.12386336556, Loss: 8.684814453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2212 , TFLOPS: 19.876251027867966, Tokens per sec: 280773.0341144811, Loss: 8.686793327331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2213 , TFLOPS: 19.86953581728137, Tokens per sec: 280678.1746740126, Loss: 8.680206298828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2214 , TFLOPS: 19.854356362417107, Tokens per sec: 280463.74884529534, Loss: 8.684924125671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2215 , TFLOPS: 19.754341735318068, Tokens per sec: 279050.93662698247, Loss: 8.684944152832031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2216 , TFLOPS: 19.85260787557616, Tokens per sec: 280439.0496223506, Loss: 8.686182975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2217 , TFLOPS: 19.844368091420407, Tokens per sec: 280322.6539703429, Loss: 8.685989379882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2218 , TFLOPS: 19.874990551497774, Tokens per sec: 280755.22855475097, Loss: 8.68571662902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2219 , TFLOPS: 19.18085151309444, Tokens per sec: 270949.78166053764, Loss: 8.687056541442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2220 , TFLOPS: 19.83062911418705, Tokens per sec: 280128.57640923583, Loss: 8.681687355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2221 , TFLOPS: 19.843646495519625, Tokens per sec: 280312.4606662744, Loss: 8.685256958007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2222 , TFLOPS: 19.846008745621006, Tokens per sec: 280345.82994337665, Loss: 8.687494277954102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2223 , TFLOPS: 19.83788449623693, Tokens per sec: 280231.0663420179, Loss: 8.686700820922852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2224 , TFLOPS: 19.832282815699195, Tokens per sec: 280151.9366893225, Loss: 8.684054374694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2225 , TFLOPS: 19.73136255471144, Tokens per sec: 278726.3314360266, Loss: 8.689698219299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2226 , TFLOPS: 19.80022644464611, Tokens per sec: 279699.1065982427, Loss: 8.684422492980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2227 , TFLOPS: 19.825788051111154, Tokens per sec: 280060.19128136366, Loss: 8.688464164733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2228 , TFLOPS: 19.851533602393253, Tokens per sec: 280423.87437926227, Loss: 8.68613052368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2229 , TFLOPS: 19.69260759458337, Tokens per sec: 278178.8767008812, Loss: 8.684832572937012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2230 , TFLOPS: 19.829760250152695, Tokens per sec: 280116.3027872728, Loss: 8.685519218444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2231 , TFLOPS: 19.914273003388264, Tokens per sec: 281310.1346680459, Loss: 8.687028884887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2232 , TFLOPS: 19.91482456622442, Tokens per sec: 281317.92607552995, Loss: 8.691078186035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2233 , TFLOPS: 19.872642990348513, Tokens per sec: 280722.0667746082, Loss: 8.68918514251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2234 , TFLOPS: 19.207976198498162, Tokens per sec: 271332.9464842021, Loss: 8.685382843017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2235 , TFLOPS: 19.6696420155445, Tokens per sec: 277854.46364643256, Loss: 8.68313217163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2236 , TFLOPS: 19.655238074530242, Tokens per sec: 277650.9927697512, Loss: 8.683592796325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2237 , TFLOPS: 19.863487162999434, Tokens per sec: 280592.73104519804, Loss: 8.686848640441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2238 , TFLOPS: 19.823898800423024, Tokens per sec: 280033.50362043787, Loss: 8.684541702270508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2239 , TFLOPS: 19.625317144667346, Tokens per sec: 277228.3280404039, Loss: 8.686599731445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2240 , TFLOPS: 19.796134173059865, Tokens per sec: 279641.298940849, Loss: 8.687111854553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2241 , TFLOPS: 19.765274644534184, Tokens per sec: 279205.3755142758, Loss: 8.682251930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2242 , TFLOPS: 19.89944572862422, Tokens per sec: 281100.6837551291, Loss: 8.684287071228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2243 , TFLOPS: 19.886510048365928, Tokens per sec: 280917.9536120372, Loss: 8.686141967773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2244 , TFLOPS: 19.659264931997235, Tokens per sec: 277707.8763836364, Loss: 8.69007396697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2245 , TFLOPS: 19.78131023751808, Tokens per sec: 279431.8951979727, Loss: 8.688364028930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2246 , TFLOPS: 19.81111612372845, Tokens per sec: 279852.9348142476, Loss: 8.686976432800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2247 , TFLOPS: 19.867051309520793, Tokens per sec: 280643.07837837614, Loss: 8.686288833618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2248 , TFLOPS: 18.799384197329697, Tokens per sec: 265561.1530145921, Loss: 8.693002700805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2249 , TFLOPS: 19.807148621548276, Tokens per sec: 279796.889656465, Loss: 8.68474006652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2250 , TFLOPS: 19.730864255288044, Tokens per sec: 278719.29243049136, Loss: 8.680255889892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2251 , TFLOPS: 19.84803407307034, Tokens per sec: 280374.4398322449, Loss: 8.682382583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2252 , TFLOPS: 19.86640444989742, Tokens per sec: 280633.94080314395, Loss: 8.684430122375488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2253 , TFLOPS: 19.811042775482658, Tokens per sec: 279851.89869282366, Loss: 8.681656837463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2254 , TFLOPS: 19.81833118053355, Tokens per sec: 279954.8551103656, Loss: 8.688421249389648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2255 , TFLOPS: 19.839205379819273, Tokens per sec: 280249.7252174067, Loss: 8.686610221862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2256 , TFLOPS: 19.752009306133314, Tokens per sec: 279017.98860181705, Loss: 8.687546730041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2257 , TFLOPS: 19.73163496134348, Tokens per sec: 278730.1794673522, Loss: 8.688535690307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2258 , TFLOPS: 19.79221477088667, Tokens per sec: 279585.93324645626, Loss: 8.687285423278809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2259 , TFLOPS: 19.864993215386143, Tokens per sec: 280614.00562547776, Loss: 8.683431625366211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2260 , TFLOPS: 19.814492459692474, Tokens per sec: 279900.62912498455, Loss: 8.684392929077148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2261 , TFLOPS: 19.789737590567107, Tokens per sec: 279550.9404586616, Loss: 8.684731483459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2262 , TFLOPS: 19.54881270026579, Tokens per sec: 276147.62197829236, Loss: 8.684795379638672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2263 , TFLOPS: 19.06514052185557, Tokens per sec: 269315.2417241593, Loss: 8.688968658447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2264 , TFLOPS: 19.808781938414203, Tokens per sec: 279819.9619818987, Loss: 8.686235427856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2265 , TFLOPS: 19.792234362426367, Tokens per sec: 279586.2099976422, Loss: 8.684976577758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2266 , TFLOPS: 19.831710920257713, Tokens per sec: 280143.85806231806, Loss: 8.685148239135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2267 , TFLOPS: 19.689708456585244, Tokens per sec: 278137.92331531027, Loss: 8.68464469909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2268 , TFLOPS: 19.71642169576022, Tokens per sec: 278515.2760265267, Loss: 8.689026832580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2269 , TFLOPS: 19.8062768785944, Tokens per sec: 279784.5753667237, Loss: 8.682281494140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2270 , TFLOPS: 19.798309000538655, Tokens per sec: 279672.02067549806, Loss: 8.683629035949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2271 , TFLOPS: 19.76007343598129, Tokens per sec: 279131.90295124514, Loss: 8.68741512298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2272 , TFLOPS: 19.831419120026872, Tokens per sec: 280139.7360759307, Loss: 8.682140350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2273 , TFLOPS: 19.70556100046042, Tokens per sec: 278361.8572370556, Loss: 8.680150985717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2274 , TFLOPS: 19.846137713109048, Tokens per sec: 280347.65174532856, Loss: 8.68524169921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2275 , TFLOPS: 19.80826014500233, Tokens per sec: 279812.5910989677, Loss: 8.683024406433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2276 , TFLOPS: 19.849206272555502, Tokens per sec: 280390.998387758, Loss: 8.681267738342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2277 , TFLOPS: 19.16021477970258, Tokens per sec: 270658.26600999973, Loss: 8.682379722595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2278 , TFLOPS: 19.8975126182456, Tokens per sec: 281073.3765297614, Loss: 8.681330680847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2279 , TFLOPS: 19.796061401081307, Tokens per sec: 279640.2709598082, Loss: 8.685845375061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2280 , TFLOPS: 19.68032291459492, Tokens per sec: 278005.34262402507, Loss: 8.683292388916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2281 , TFLOPS: 19.88212738543028, Tokens per sec: 280856.0438701941, Loss: 8.682559967041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2282 , TFLOPS: 19.808277837039597, Tokens per sec: 279812.84101767605, Loss: 8.686243057250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2283 , TFLOPS: 19.881467570969086, Tokens per sec: 280846.72329419805, Loss: 8.687150001525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2284 , TFLOPS: 19.87372295498777, Tokens per sec: 280737.3224155274, Loss: 8.683079719543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2285 , TFLOPS: 19.767474209249826, Tokens per sec: 279236.44668851665, Loss: 8.686909675598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2286 , TFLOPS: 19.854492935000504, Tokens per sec: 280465.67807724973, Loss: 8.687292098999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2287 , TFLOPS: 19.85446435983066, Tokens per sec: 280465.27442280174, Loss: 8.686901092529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2288 , TFLOPS: 19.716915974202163, Tokens per sec: 278522.2582314529, Loss: 8.682680130004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2289 , TFLOPS: 19.840011413008916, Tokens per sec: 280261.11128734186, Loss: 8.68489933013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2290 , TFLOPS: 19.730770482324676, Tokens per sec: 278717.96778836416, Loss: 8.684638977050781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2291 , TFLOPS: 19.87673674147388, Tokens per sec: 280779.8953320544, Loss: 8.683671951293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2292 , TFLOPS: 19.25243157906151, Tokens per sec: 271960.9256773601, Loss: 8.68741226196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2293 , TFLOPS: 19.848572643173508, Tokens per sec: 280382.0477036558, Loss: 8.680830001831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2294 , TFLOPS: 19.77858237951022, Tokens per sec: 279393.36132312746, Loss: 8.68221378326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2295 , TFLOPS: 19.870733709666506, Tokens per sec: 280695.09616886824, Loss: 8.68603801727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2296 , TFLOPS: 19.902280243355378, Tokens per sec: 281140.7242688153, Loss: 8.68283462524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2297 , TFLOPS: 19.874043824060987, Tokens per sec: 280741.85503000865, Loss: 8.685836791992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2298 , TFLOPS: 19.85711325911439, Tokens per sec: 280502.6929222921, Loss: 8.681439399719238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2299 , TFLOPS: 19.672087421053785, Tokens per sec: 277889.00758147944, Loss: 8.683822631835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2300 , TFLOPS: 19.851396032849884, Tokens per sec: 280421.9310641955, Loss: 8.681694984436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2301 , TFLOPS: 19.795371085289382, Tokens per sec: 279630.5195203077, Loss: 8.686968803405762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2302 , TFLOPS: 19.866177512199446, Tokens per sec: 280630.735068525, Loss: 8.68613338470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2303 , TFLOPS: 19.787836146742528, Tokens per sec: 279524.08055680804, Loss: 8.685662269592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2304 , TFLOPS: 19.746566025804892, Tokens per sec: 278941.0965193411, Loss: 8.685166358947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2305 , TFLOPS: 19.70731377170513, Tokens per sec: 278386.61698172765, Loss: 8.685901641845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2306 , TFLOPS: 19.528695659938627, Tokens per sec: 275863.4475410637, Loss: 8.683959007263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2307 , TFLOPS: 19.73384875946126, Tokens per sec: 278761.45170342806, Loss: 8.682302474975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2308 , TFLOPS: 19.75824400732075, Tokens per sec: 279106.0603396281, Loss: 8.685327529907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2309 , TFLOPS: 19.810807637895746, Tokens per sec: 279848.57712611475, Loss: 8.687952995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2310 , TFLOPS: 19.756027594177194, Tokens per sec: 279074.7511635519, Loss: 8.684464454650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2311 , TFLOPS: 19.8682192073683, Tokens per sec: 280659.57616871496, Loss: 8.686200141906738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2312 , TFLOPS: 19.90234747903231, Tokens per sec: 281141.6740437495, Loss: 8.687387466430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2313 , TFLOPS: 19.90167598800688, Tokens per sec: 281132.18852395366, Loss: 8.685050010681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2314 , TFLOPS: 19.858639371404124, Tokens per sec: 280524.2508698843, Loss: 8.684946060180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2315 , TFLOPS: 19.86036566427866, Tokens per sec: 280548.6365796177, Loss: 8.685290336608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2316 , TFLOPS: 19.787838107723484, Tokens per sec: 279524.1082577347, Loss: 8.684779167175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2317 , TFLOPS: 19.803652849711504, Tokens per sec: 279747.50818790705, Loss: 8.684703826904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2318 , TFLOPS: 19.87202123877211, Tokens per sec: 280713.2838770524, Loss: 8.682655334472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2319 , TFLOPS: 19.886212665432826, Tokens per sec: 280913.7527640853, Loss: 8.685196876525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2320 , TFLOPS: 19.877301240983083, Tokens per sec: 280787.8694837057, Loss: 8.684563636779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2321 , TFLOPS: 19.279988584072935, Tokens per sec: 272350.19747199165, Loss: 8.684470176696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2322 , TFLOPS: 19.864368818524134, Tokens per sec: 280605.18536046945, Loss: 8.684349060058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2323 , TFLOPS: 19.819388329032787, Tokens per sec: 279969.7884492143, Loss: 8.683894157409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2324 , TFLOPS: 19.80539300030242, Tokens per sec: 279772.0896525174, Loss: 8.682780265808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2325 , TFLOPS: 19.66213329458868, Tokens per sec: 277748.39503409073, Loss: 8.68256664276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2326 , TFLOPS: 19.8427990824369, Tokens per sec: 280300.4900616543, Loss: 8.686266899108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2327 , TFLOPS: 19.896640738250237, Tokens per sec: 281061.0603041645, Loss: 8.68362045288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2328 , TFLOPS: 19.91321640950355, Tokens per sec: 281295.20916371304, Loss: 8.685467720031738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2329 , TFLOPS: 19.897850289065243, Tokens per sec: 281078.1464866527, Loss: 8.684829711914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2330 , TFLOPS: 19.906284117412493, Tokens per sec: 281197.28321776656, Loss: 8.681414604187012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2331 , TFLOPS: 19.907789948370244, Tokens per sec: 281218.5546701284, Loss: 8.681442260742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2332 , TFLOPS: 19.904651872293073, Tokens per sec: 281174.226031884, Loss: 8.689335823059082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2333 , TFLOPS: 19.86270720897228, Tokens per sec: 280581.7133709714, Loss: 8.680130004882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2334 , TFLOPS: 19.880317704554187, Tokens per sec: 280830.48021688504, Loss: 8.683614730834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2335 , TFLOPS: 19.877034174232346, Tokens per sec: 280784.09688384214, Loss: 8.68510913848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2336 , TFLOPS: 19.77388951319894, Tokens per sec: 279327.0696310441, Loss: 8.684687614440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2337 , TFLOPS: 19.387698714630616, Tokens per sec: 273871.7168027319, Loss: 8.684986114501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2338 , TFLOPS: 19.870210740244406, Tokens per sec: 280687.708673551, Loss: 8.678715705871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2339 , TFLOPS: 19.904071336454614, Tokens per sec: 281166.02535014844, Loss: 8.683070182800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2340 , TFLOPS: 19.902713550906565, Tokens per sec: 281146.84519552725, Loss: 8.676785469055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2341 , TFLOPS: 19.903230392424692, Tokens per sec: 281154.14612773014, Loss: 8.6823148727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2342 , TFLOPS: 19.8714566041608, Tokens per sec: 280705.30781694123, Loss: 8.682771682739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2343 , TFLOPS: 19.906718970129834, Tokens per sec: 281203.4259715821, Loss: 8.685410499572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2344 , TFLOPS: 19.888579846854878, Tokens per sec: 280947.1917012992, Loss: 8.682222366333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2345 , TFLOPS: 19.889328331579314, Tokens per sec: 280957.764838395, Loss: 8.685230255126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2346 , TFLOPS: 19.848895277185377, Tokens per sec: 280386.60524975904, Loss: 8.683223724365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2347 , TFLOPS: 19.88818383887725, Tokens per sec: 280941.5976704502, Loss: 8.682385444641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2348 , TFLOPS: 19.841857993983105, Tokens per sec: 280287.19619350124, Loss: 8.686258316040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2349 , TFLOPS: 19.857033537140833, Tokens per sec: 280501.5667652331, Loss: 8.68340015411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2350 , TFLOPS: 19.86445135654967, Tokens per sec: 280606.3512972362, Loss: 8.68471908569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2351 , TFLOPS: 19.788080180586817, Tokens per sec: 279527.5277925464, Loss: 8.680537223815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2352 , TFLOPS: 19.14641387235675, Tokens per sec: 270463.3136206589, Loss: 8.683633804321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2353 , TFLOPS: 19.880933343149525, Tokens per sec: 280839.17676212837, Loss: 8.683300018310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2354 , TFLOPS: 19.851100662225438, Tokens per sec: 280417.7586422283, Loss: 8.682638168334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2355 , TFLOPS: 19.724735337523736, Tokens per sec: 278632.7149952341, Loss: 8.680683135986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2356 , TFLOPS: 19.864544930832267, Tokens per sec: 280607.6731327878, Loss: 8.685070991516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2357 , TFLOPS: 19.782585321585685, Tokens per sec: 279449.9071068539, Loss: 8.680792808532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2358 , TFLOPS: 19.833534908662667, Tokens per sec: 280169.6238245803, Loss: 8.686487197875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2359 , TFLOPS: 19.823046979853814, Tokens per sec: 280021.470755417, Loss: 8.686153411865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2360 , TFLOPS: 19.73635216246085, Tokens per sec: 278796.814914268, Loss: 8.682939529418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2361 , TFLOPS: 19.84701408704604, Tokens per sec: 280360.0314525965, Loss: 8.681048393249512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2362 , TFLOPS: 19.863221105601365, Tokens per sec: 280588.9727035068, Loss: 8.683026313781738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2363 , TFLOPS: 19.866585407393973, Tokens per sec: 280636.497018865, Loss: 8.684611320495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2364 , TFLOPS: 19.86023360647976, Tokens per sec: 280546.77112376166, Loss: 8.6867036819458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2365 , TFLOPS: 19.520479526861322, Tokens per sec: 275747.38598551333, Loss: 8.68271255493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2366 , TFLOPS: 19.808763516145216, Tokens per sec: 279819.70174789627, Loss: 8.68354320526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2367 , TFLOPS: 19.167477680964844, Tokens per sec: 270760.8621595966, Loss: 8.680411338806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2368 , TFLOPS: 19.741978011485482, Tokens per sec: 278876.2859723614, Loss: 8.68575382232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2369 , TFLOPS: 19.916271172292433, Tokens per sec: 281338.36091378564, Loss: 8.683670997619629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2370 , TFLOPS: 19.860964671058312, Tokens per sec: 280557.09818290203, Loss: 8.684587478637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2371 , TFLOPS: 19.81108645745152, Tokens per sec: 279852.51574675983, Loss: 8.683977127075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2372 , TFLOPS: 19.756630299035834, Tokens per sec: 279083.26500610687, Loss: 8.685973167419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2373 , TFLOPS: 19.827784997191568, Tokens per sec: 280088.40025342663, Loss: 8.683440208435059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2374 , TFLOPS: 19.718631164504842, Tokens per sec: 278546.48710563505, Loss: 8.683488845825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2375 , TFLOPS: 19.637139775243625, Tokens per sec: 277395.3351814117, Loss: 8.681553840637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2376 , TFLOPS: 19.582336504658226, Tokens per sec: 276621.1811148281, Loss: 8.683243751525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2377 , TFLOPS: 19.791962208092617, Tokens per sec: 279582.36553029635, Loss: 8.682982444763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2378 , TFLOPS: 19.813846335718942, Tokens per sec: 279891.5019415832, Loss: 8.679976463317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2379 , TFLOPS: 19.823010197233316, Tokens per sec: 280020.9511620618, Loss: 8.686158180236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2380 , TFLOPS: 19.79433981523884, Tokens per sec: 279615.9517418732, Loss: 8.68187427520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2381 , TFLOPS: 19.13184770248556, Tokens per sec: 270257.55109006766, Loss: 8.680154800415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2382 , TFLOPS: 19.842754643894203, Tokens per sec: 280299.8623203138, Loss: 8.687275886535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2383 , TFLOPS: 19.75843092118008, Tokens per sec: 279108.700695263, Loss: 8.682857513427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2384 , TFLOPS: 19.868157168195438, Tokens per sec: 280658.699799888, Loss: 8.68608570098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2385 , TFLOPS: 19.82321264872799, Tokens per sec: 280023.811003203, Loss: 8.685195922851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2386 , TFLOPS: 19.857582076298055, Tokens per sec: 280509.3154600593, Loss: 8.68304443359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2387 , TFLOPS: 19.70114885370042, Tokens per sec: 278299.5310050601, Loss: 8.686602592468262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2388 , TFLOPS: 19.773803264546974, Tokens per sec: 279325.8512777607, Loss: 8.68323040008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2389 , TFLOPS: 19.75926212597512, Tokens per sec: 279120.4423407044, Loss: 8.682180404663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2390 , TFLOPS: 19.63292074805889, Tokens per sec: 277335.7369673416, Loss: 8.685978889465332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2391 , TFLOPS: 19.825611644480293, Tokens per sec: 280057.6993514261, Loss: 8.683247566223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2392 , TFLOPS: 19.668915693094053, Tokens per sec: 277844.20357485913, Loss: 8.682868003845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2393 , TFLOPS: 19.636852962553196, Tokens per sec: 277391.28364929924, Loss: 8.67924976348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2394 , TFLOPS: 19.808608008734584, Tokens per sec: 279817.5050415129, Loss: 8.685269355773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2395 , TFLOPS: 19.76802370950209, Tokens per sec: 279244.20895982516, Loss: 8.682901382446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2396 , TFLOPS: 19.264283137110343, Tokens per sec: 272128.3414494636, Loss: 8.686978340148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2397 , TFLOPS: 19.833379973375852, Tokens per sec: 280167.43520004995, Loss: 8.682251930236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2398 , TFLOPS: 19.678925729177447, Tokens per sec: 277985.60590464406, Loss: 8.683602333068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2399 , TFLOPS: 19.738537335150074, Tokens per sec: 278827.68278593844, Loss: 8.681410789489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2400 , TFLOPS: 19.83258535035297, Tokens per sec: 280156.21031076886, Loss: 8.68349838256836 +------------------------------------------------------------------ + eval ppl=5871.34130859375, eval loss=8.677838325500488 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002400, took 0.16s +Deleting checkpoint /work/llm_recipes/models/tiny-mistral-sample2/iter_0000400 +[rank0]:[2024-08-03 21:21:50,782] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.001019665000058012, 'preprocessing_with_comm': 0.0008134670006256783, 'state_converting': 0.02897009899970726, : 0.0311177259991382}) +------------------------------------------------------------------ +iteration: 2401 , TFLOPS: 19.681544246264927, Tokens per sec: 278022.59522353025, Loss: 8.681418418884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2402 , TFLOPS: 19.782768298105584, Tokens per sec: 279452.49184341164, Loss: 8.684545516967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2403 , TFLOPS: 19.79866907614764, Tokens per sec: 279677.1071236971, Loss: 8.68531322479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2404 , TFLOPS: 19.756512588540247, Tokens per sec: 279081.60222106025, Loss: 8.682826042175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2405 , TFLOPS: 19.698697533724708, Tokens per sec: 278264.9035219296, Loss: 8.682825088500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2406 , TFLOPS: 19.64617508512554, Tokens per sec: 277522.96847433766, Loss: 8.680678367614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2407 , TFLOPS: 19.860384196235884, Tokens per sec: 280548.8983630823, Loss: 8.685952186584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2408 , TFLOPS: 19.872619446786356, Tokens per sec: 280721.73419693194, Loss: 8.679607391357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2409 , TFLOPS: 19.776318814654953, Tokens per sec: 279361.38607933343, Loss: 8.67982292175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2410 , TFLOPS: 18.946437572831105, Tokens per sec: 267638.4372246988, Loss: 8.683914184570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2411 , TFLOPS: 19.86051430779067, Tokens per sec: 280550.73632618506, Loss: 8.679112434387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2412 , TFLOPS: 19.781681743195154, Tokens per sec: 279437.14310795313, Loss: 8.680675506591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2413 , TFLOPS: 19.868299576265628, Tokens per sec: 280660.71146425535, Loss: 8.686136245727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2414 , TFLOPS: 19.834905642299706, Tokens per sec: 280188.9868846309, Loss: 8.681715965270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2415 , TFLOPS: 19.78744783989357, Tokens per sec: 279518.5953125335, Loss: 8.68506145477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2416 , TFLOPS: 19.862440323752054, Tokens per sec: 280577.94333541795, Loss: 8.684520721435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2417 , TFLOPS: 19.86461162655708, Tokens per sec: 280608.61528032814, Loss: 8.684633255004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2418 , TFLOPS: 19.85029861032496, Tokens per sec: 280406.4288122066, Loss: 8.682633399963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2419 , TFLOPS: 19.759284971688565, Tokens per sec: 279120.7650605308, Loss: 8.682259559631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2420 , TFLOPS: 19.861853265339995, Tokens per sec: 280569.6505154428, Loss: 8.68324089050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2421 , TFLOPS: 19.71004589372281, Tokens per sec: 278425.2110902141, Loss: 8.682252883911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2422 , TFLOPS: 19.753223791178907, Tokens per sec: 279035.14448552264, Loss: 8.684553146362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2423 , TFLOPS: 19.827904553124878, Tokens per sec: 280090.08910722984, Loss: 8.681386947631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2424 , TFLOPS: 19.86228599570566, Tokens per sec: 280575.7632887998, Loss: 8.683886528015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2425 , TFLOPS: 19.276005050017147, Tokens per sec: 272293.9258470371, Loss: 8.687006950378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2426 , TFLOPS: 19.869639250856967, Tokens per sec: 280679.6357824887, Loss: 8.681805610656738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2427 , TFLOPS: 19.873157074986004, Tokens per sec: 280729.32876296097, Loss: 8.679685592651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2428 , TFLOPS: 19.776775121058584, Tokens per sec: 279367.83188912185, Loss: 8.686318397521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2429 , TFLOPS: 19.87005944425342, Tokens per sec: 280685.57145791716, Loss: 8.68427562713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2430 , TFLOPS: 19.87471655841881, Tokens per sec: 280751.35811318737, Loss: 8.6849365234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2431 , TFLOPS: 19.86671560700945, Tokens per sec: 280638.336225918, Loss: 8.683720588684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2432 , TFLOPS: 19.867423257600677, Tokens per sec: 280648.3325377634, Loss: 8.680217742919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2433 , TFLOPS: 19.62297196091526, Tokens per sec: 277195.1998435068, Loss: 8.67994213104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2434 , TFLOPS: 19.64034854314206, Tokens per sec: 277440.6624162785, Loss: 8.6790189743042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2435 , TFLOPS: 19.865016937834376, Tokens per sec: 280614.3407301088, Loss: 8.682981491088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2436 , TFLOPS: 19.852496791815017, Tokens per sec: 280437.4804469255, Loss: 8.683709144592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2437 , TFLOPS: 19.716401136760673, Tokens per sec: 278514.9856089488, Loss: 8.680536270141602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2438 , TFLOPS: 19.89599449830968, Tokens per sec: 281051.93148260645, Loss: 8.683536529541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2439 , TFLOPS: 19.751621014265115, Tokens per sec: 279012.5035691616, Loss: 8.680360794067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2440 , TFLOPS: 19.187082820043795, Tokens per sec: 271037.8054511517, Loss: 8.681865692138672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2441 , TFLOPS: 19.752257768300932, Tokens per sec: 279021.4983922985, Loss: 8.683144569396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2442 , TFLOPS: 19.69927603533694, Tokens per sec: 278273.0754680717, Loss: 8.683222770690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2443 , TFLOPS: 19.8497258856158, Tokens per sec: 280398.3384709206, Loss: 8.679931640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2444 , TFLOPS: 19.894049076022995, Tokens per sec: 281024.45033853647, Loss: 8.680376052856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2445 , TFLOPS: 19.872934570871443, Tokens per sec: 280726.1856573894, Loss: 8.681175231933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2446 , TFLOPS: 19.89061514626832, Tokens per sec: 280975.9424546904, Loss: 8.677713394165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2447 , TFLOPS: 19.91431226857712, Tokens per sec: 281310.6893303006, Loss: 8.682888984680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2448 , TFLOPS: 19.910999310271965, Tokens per sec: 281263.8902959189, Loss: 8.681077003479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2449 , TFLOPS: 19.90915958004741, Tokens per sec: 281237.9021638323, Loss: 8.683964729309082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2450 , TFLOPS: 19.898014648556604, Tokens per sec: 281080.46823802375, Loss: 8.681988716125488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2451 , TFLOPS: 19.899091157588693, Tokens per sec: 281095.67506484274, Loss: 8.681873321533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2452 , TFLOPS: 19.906276793063725, Tokens per sec: 281197.17975360603, Loss: 8.687531471252441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2453 , TFLOPS: 19.910162488146053, Tokens per sec: 281252.0692997473, Loss: 8.681962966918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2454 , TFLOPS: 19.90657865691453, Tokens per sec: 281201.4438992521, Loss: 8.683119773864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2455 , TFLOPS: 19.30888916038551, Tokens per sec: 272758.4486299973, Loss: 8.681899070739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2456 , TFLOPS: 19.915189322541647, Tokens per sec: 281323.0786436755, Loss: 8.678343772888184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2457 , TFLOPS: 19.908483939016076, Tokens per sec: 281228.35802082083, Loss: 8.680055618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2458 , TFLOPS: 19.91691302278967, Tokens per sec: 281347.42772982176, Loss: 8.68319320678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2459 , TFLOPS: 19.915156722666755, Tokens per sec: 281322.6181360188, Loss: 8.67707347869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2460 , TFLOPS: 19.916474652584675, Tokens per sec: 281341.2352877788, Loss: 8.68257999420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2461 , TFLOPS: 19.91548333381465, Tokens per sec: 281327.2318633681, Loss: 8.681682586669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2462 , TFLOPS: 19.917224159712582, Tokens per sec: 281351.82286740316, Loss: 8.685564994812012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2463 , TFLOPS: 19.90699488296327, Tokens per sec: 281207.32353170484, Loss: 8.680961608886719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2464 , TFLOPS: 19.910407406243543, Tokens per sec: 281255.5290264958, Loss: 8.682781219482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2465 , TFLOPS: 19.911267263224012, Tokens per sec: 281267.6754143117, Loss: 8.68088436126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2466 , TFLOPS: 19.878590770671128, Tokens per sec: 280806.0854522294, Loss: 8.682256698608398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2467 , TFLOPS: 19.685524435142867, Tokens per sec: 278078.8195943153, Loss: 8.68186092376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2468 , TFLOPS: 19.851771769591565, Tokens per sec: 280427.2387424313, Loss: 8.684659004211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2469 , TFLOPS: 19.258258967446903, Tokens per sec: 272043.2436917399, Loss: 8.680276870727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2470 , TFLOPS: 19.856083824176288, Tokens per sec: 280488.1510667586, Loss: 8.686965942382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2471 , TFLOPS: 19.713826389693086, Tokens per sec: 278478.6145878125, Loss: 8.683603286743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2472 , TFLOPS: 19.85171201022675, Tokens per sec: 280426.39457830036, Loss: 8.68053150177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2473 , TFLOPS: 19.851065464098987, Tokens per sec: 280417.2614315247, Loss: 8.684898376464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2474 , TFLOPS: 19.844701209155318, Tokens per sec: 280327.3596101044, Loss: 8.683364868164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2475 , TFLOPS: 19.85253137774622, Tokens per sec: 280437.9690097248, Loss: 8.681375503540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2476 , TFLOPS: 19.89005745460077, Tokens per sec: 280968.06447099114, Loss: 8.681051254272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2477 , TFLOPS: 19.887202753526147, Tokens per sec: 280927.7388038879, Loss: 8.679018020629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2478 , TFLOPS: 19.897525591482545, Tokens per sec: 281073.5597904324, Loss: 8.686516761779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2479 , TFLOPS: 19.905324955230284, Tokens per sec: 281183.7340391147, Loss: 8.681330680847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2480 , TFLOPS: 19.90778743152676, Tokens per sec: 281218.5191170565, Loss: 8.68710994720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2481 , TFLOPS: 19.90566855124118, Tokens per sec: 281188.5876955891, Loss: 8.678889274597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2482 , TFLOPS: 19.90636944560014, Tokens per sec: 281198.4885685195, Loss: 8.68114948272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2483 , TFLOPS: 19.898489337767053, Tokens per sec: 281087.17372436984, Loss: 8.679165840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2484 , TFLOPS: 19.32106047642447, Tokens per sec: 272930.3813213602, Loss: 8.682042121887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2485 , TFLOPS: 19.782126645758886, Tokens per sec: 279443.42782646115, Loss: 8.682455062866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2486 , TFLOPS: 19.860444157171084, Tokens per sec: 280549.7453746078, Loss: 8.680994987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2487 , TFLOPS: 19.756098238335863, Tokens per sec: 279075.7490868898, Loss: 8.680400848388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2488 , TFLOPS: 19.724146012885914, Tokens per sec: 278624.390162427, Loss: 8.680142402648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2489 , TFLOPS: 19.763539916247424, Tokens per sec: 279180.8706454374, Loss: 8.682483673095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2490 , TFLOPS: 19.720500088580206, Tokens per sec: 278572.8876316917, Loss: 8.679985046386719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2491 , TFLOPS: 19.862548673923076, Tokens per sec: 280579.4738959961, Loss: 8.68018913269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2492 , TFLOPS: 19.8567360647793, Tokens per sec: 280497.3646540103, Loss: 8.684861183166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2493 , TFLOPS: 19.865733840253405, Tokens per sec: 280624.4677337914, Loss: 8.681114196777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2494 , TFLOPS: 19.85062030356949, Tokens per sec: 280410.9730689779, Loss: 8.679344177246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2495 , TFLOPS: 19.753926851598056, Tokens per sec: 279045.0759563404, Loss: 8.683968544006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2496 , TFLOPS: 19.679402259735596, Tokens per sec: 277992.33740197076, Loss: 8.682994842529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2497 , TFLOPS: 19.879848071702263, Tokens per sec: 280823.8461569424, Loss: 8.684351921081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2498 , TFLOPS: 19.07102186999665, Tokens per sec: 269398.3219770671, Loss: 8.678975105285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2499 , TFLOPS: 19.723437003250652, Tokens per sec: 278614.3746526495, Loss: 8.68254280090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2500 , TFLOPS: 19.79451557812042, Tokens per sec: 279618.4345781725, Loss: 8.680609703063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2501 , TFLOPS: 19.738193932904792, Tokens per sec: 278822.8318666079, Loss: 8.680842399597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2502 , TFLOPS: 19.911258522846413, Tokens per sec: 281267.55194724875, Loss: 8.681309700012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2503 , TFLOPS: 19.907422809887333, Tokens per sec: 281213.3684513771, Loss: 8.680168151855469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2504 , TFLOPS: 19.76699610047173, Tokens per sec: 279229.6928971666, Loss: 8.680449485778809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2505 , TFLOPS: 19.755186009038738, Tokens per sec: 279062.86288479855, Loss: 8.67971420288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2506 , TFLOPS: 19.870915316865588, Tokens per sec: 280697.6615623235, Loss: 8.683085441589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2507 , TFLOPS: 19.84642146647426, Tokens per sec: 280351.6600612418, Loss: 8.689745903015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2508 , TFLOPS: 19.90568129644699, Tokens per sec: 281188.76773507945, Loss: 8.675782203674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2509 , TFLOPS: 19.795412782896513, Tokens per sec: 279631.108543039, Loss: 8.6788330078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2510 , TFLOPS: 19.904601517341636, Tokens per sec: 281173.5147150226, Loss: 8.679800033569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2511 , TFLOPS: 19.796719511255265, Tokens per sec: 279649.56746094744, Loss: 8.684751510620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2512 , TFLOPS: 19.904202430147816, Tokens per sec: 281167.87718699337, Loss: 8.681500434875488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2513 , TFLOPS: 19.556662741028173, Tokens per sec: 276258.5120933189, Loss: 8.68274211883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2514 , TFLOPS: 19.8875178229596, Tokens per sec: 280932.18949233333, Loss: 8.680479049682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2515 , TFLOPS: 19.89516929126761, Tokens per sec: 281040.274561759, Loss: 8.684164047241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2516 , TFLOPS: 19.791008355106552, Tokens per sec: 279568.8913496462, Loss: 8.67928695678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2517 , TFLOPS: 19.913907140879374, Tokens per sec: 281304.966473775, Loss: 8.679431915283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2518 , TFLOPS: 19.917808480840122, Tokens per sec: 281360.07702034415, Loss: 8.67946720123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2519 , TFLOPS: 19.89458034429237, Tokens per sec: 281031.9550638346, Loss: 8.680246353149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2520 , TFLOPS: 19.77628666783861, Tokens per sec: 279360.9319716071, Loss: 8.678598403930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2521 , TFLOPS: 19.90874275185478, Tokens per sec: 281232.0140254612, Loss: 8.681890487670898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2522 , TFLOPS: 19.90616669685691, Tokens per sec: 281195.62452842866, Loss: 8.678596496582031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2523 , TFLOPS: 19.89469892923698, Tokens per sec: 281033.6302013972, Loss: 8.675243377685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2524 , TFLOPS: 19.903036611210002, Tokens per sec: 281151.40876344923, Loss: 8.684454917907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2525 , TFLOPS: 19.857317520730238, Tokens per sec: 280505.5783333062, Loss: 8.686352729797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2526 , TFLOPS: 19.869784792129117, Tokens per sec: 280681.6917066422, Loss: 8.679795265197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2527 , TFLOPS: 19.53790662661163, Tokens per sec: 275993.5621716487, Loss: 8.67951774597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2528 , TFLOPS: 19.90995097746558, Tokens per sec: 281249.08148804074, Loss: 8.683170318603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2529 , TFLOPS: 19.642057840844124, Tokens per sec: 277464.8080512547, Loss: 8.679768562316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2530 , TFLOPS: 19.88738632273771, Tokens per sec: 280930.33191284136, Loss: 8.68034839630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2531 , TFLOPS: 19.80549152764234, Tokens per sec: 279773.48145523376, Loss: 8.681167602539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2532 , TFLOPS: 19.907761424193733, Tokens per sec: 281218.15173601505, Loss: 8.679693222045898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2533 , TFLOPS: 19.856809989250713, Tokens per sec: 280498.4089152296, Loss: 8.680755615234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2534 , TFLOPS: 19.860166627893914, Tokens per sec: 280545.8249805105, Loss: 8.680630683898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2535 , TFLOPS: 19.890135617248394, Tokens per sec: 280969.16860092484, Loss: 8.683650970458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2536 , TFLOPS: 19.84844767142192, Tokens per sec: 280380.2823457005, Loss: 8.679713249206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2537 , TFLOPS: 19.86335896810012, Tokens per sec: 280590.9201568778, Loss: 8.681723594665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2538 , TFLOPS: 19.73092611672983, Tokens per sec: 278720.16628867923, Loss: 8.68298053741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2539 , TFLOPS: 19.61054663760946, Tokens per sec: 277019.67903128057, Loss: 8.679141998291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2540 , TFLOPS: 19.767286767841636, Tokens per sec: 279233.79888069705, Loss: 8.682944297790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2541 , TFLOPS: 19.84627606388573, Tokens per sec: 280349.6060961408, Loss: 8.68018913269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2542 , TFLOPS: 18.80844806737456, Tokens per sec: 265689.1897499779, Loss: 8.675094604492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2543 , TFLOPS: 19.86975543909597, Tokens per sec: 280681.27706405363, Loss: 8.680826187133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2544 , TFLOPS: 19.837674802874375, Tokens per sec: 280228.1042018458, Loss: 8.678292274475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2545 , TFLOPS: 19.757208278414467, Tokens per sec: 279091.429575148, Loss: 8.679832458496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2546 , TFLOPS: 19.83578950915469, Tokens per sec: 280201.4724372776, Loss: 8.677262306213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2547 , TFLOPS: 19.87035978256935, Tokens per sec: 280689.81405377213, Loss: 8.684074401855469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2548 , TFLOPS: 19.846684145534276, Tokens per sec: 280355.37068034074, Loss: 8.68002700805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2549 , TFLOPS: 19.871686081845475, Tokens per sec: 280708.54943155445, Loss: 8.685495376586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2550 , TFLOPS: 19.869618479797612, Tokens per sec: 280679.3423693421, Loss: 8.678351402282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2551 , TFLOPS: 19.86223691406147, Tokens per sec: 280575.06995874783, Loss: 8.684881210327148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2552 , TFLOPS: 19.741290020367273, Tokens per sec: 278866.5673713319, Loss: 8.677657127380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2553 , TFLOPS: 19.74927854112082, Tokens per sec: 278979.4136624617, Loss: 8.681558609008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2554 , TFLOPS: 19.903852918988918, Tokens per sec: 281162.9399728073, Loss: 8.678460121154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2555 , TFLOPS: 19.690170569978743, Tokens per sec: 278144.4511549625, Loss: 8.678418159484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2556 , TFLOPS: 19.86817718210937, Tokens per sec: 280658.982517556, Loss: 8.68211841583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2557 , TFLOPS: 19.507108452433307, Tokens per sec: 275558.5054400191, Loss: 8.67789077758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2558 , TFLOPS: 19.828032507312145, Tokens per sec: 280091.89659523836, Loss: 8.681358337402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2559 , TFLOPS: 19.730141155833117, Tokens per sec: 278709.07788713305, Loss: 8.683897018432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2560 , TFLOPS: 19.869525645890217, Tokens per sec: 280678.030992372, Loss: 8.681685447692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2561 , TFLOPS: 19.808251062645212, Tokens per sec: 279812.46280108293, Loss: 8.681180000305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2562 , TFLOPS: 19.86552062368396, Tokens per sec: 280621.45582460176, Loss: 8.681153297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2563 , TFLOPS: 19.865102795508, Tokens per sec: 280615.5535604103, Loss: 8.683014869689941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2564 , TFLOPS: 19.85813811796065, Tokens per sec: 280517.1701356962, Loss: 8.67779541015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2565 , TFLOPS: 19.90026856424612, Tokens per sec: 281112.30717716255, Loss: 8.681406021118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2566 , TFLOPS: 19.789314687905424, Tokens per sec: 279544.9665120008, Loss: 8.682036399841309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2567 , TFLOPS: 19.693673272524517, Tokens per sec: 278193.9305271036, Loss: 8.681633949279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2568 , TFLOPS: 19.9040589410419, Tokens per sec: 281165.85025185486, Loss: 8.680908203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2569 , TFLOPS: 19.807694570098583, Tokens per sec: 279804.60175623297, Loss: 8.680546760559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2570 , TFLOPS: 19.85486313750891, Tokens per sec: 280470.90757859667, Loss: 8.681859970092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2571 , TFLOPS: 19.74227240419273, Tokens per sec: 278880.44458021555, Loss: 8.68027400970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2572 , TFLOPS: 19.196263280291372, Tokens per sec: 271167.48914626427, Loss: 8.68056869506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2573 , TFLOPS: 19.795936368074273, Tokens per sec: 279638.50473655516, Loss: 8.67986011505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2574 , TFLOPS: 19.778953291257782, Tokens per sec: 279398.6008432264, Loss: 8.681207656860352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2575 , TFLOPS: 19.861878989909634, Tokens per sec: 280570.0139021528, Loss: 8.68264102935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2576 , TFLOPS: 19.870235392480463, Tokens per sec: 280688.0569124169, Loss: 8.685393333435059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2577 , TFLOPS: 19.73401340320234, Tokens per sec: 278763.7774701265, Loss: 8.679521560668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2578 , TFLOPS: 19.658913061822336, Tokens per sec: 277702.9058458583, Loss: 8.681983947753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2579 , TFLOPS: 19.781032511375166, Tokens per sec: 279427.9720229389, Loss: 8.6819429397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2580 , TFLOPS: 19.60705922150753, Tokens per sec: 276970.41559630056, Loss: 8.682331085205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2581 , TFLOPS: 19.843072745088055, Tokens per sec: 280304.3558355762, Loss: 8.679359436035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2582 , TFLOPS: 19.77899327018342, Tokens per sec: 279399.16558776685, Loss: 8.679410934448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2583 , TFLOPS: 19.690451749226057, Tokens per sec: 278148.4231087423, Loss: 8.679409980773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2584 , TFLOPS: 19.79513510570862, Tokens per sec: 279627.1860595466, Loss: 8.680116653442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2585 , TFLOPS: 19.72786283708878, Tokens per sec: 278676.8942290764, Loss: 8.677896499633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2586 , TFLOPS: 19.795800087305793, Tokens per sec: 279636.57962679456, Loss: 8.68012523651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2587 , TFLOPS: 19.266117140445324, Tokens per sec: 272154.24868318776, Loss: 8.67463207244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2588 , TFLOPS: 19.798446806705208, Tokens per sec: 279673.96733311715, Loss: 8.685101509094238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2589 , TFLOPS: 19.85362864694069, Tokens per sec: 280453.46909575915, Loss: 8.681419372558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2590 , TFLOPS: 19.861438130490416, Tokens per sec: 280563.78629733046, Loss: 8.681121826171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2591 , TFLOPS: 19.56464761880912, Tokens per sec: 276371.30692361394, Loss: 8.682002067565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2592 , TFLOPS: 19.84046070462885, Tokens per sec: 280267.4580059034, Loss: 8.676095008850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2593 , TFLOPS: 19.72788455194996, Tokens per sec: 278677.2009744204, Loss: 8.677403450012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2594 , TFLOPS: 19.86398700242526, Tokens per sec: 280599.7918048938, Loss: 8.678913116455078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2595 , TFLOPS: 19.873906078302557, Tokens per sec: 280739.90922571806, Loss: 8.67983627319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2596 , TFLOPS: 19.849828814683924, Tokens per sec: 280399.79245269584, Loss: 8.681522369384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2597 , TFLOPS: 19.889387854696896, Tokens per sec: 280958.6056652843, Loss: 8.676342964172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2598 , TFLOPS: 19.813752447972515, Tokens per sec: 279890.17567802407, Loss: 8.677635192871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2599 , TFLOPS: 19.698230686061585, Tokens per sec: 278258.3088057197, Loss: 8.679994583129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2600 , TFLOPS: 19.859062086116456, Tokens per sec: 280530.22216156195, Loss: 8.67810344696045 +------------------------------------------------------------------ + eval ppl=5904.08203125, eval loss=8.683399200439453 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002600, took 0.23s +Deleting checkpoint /work/llm_recipes/models/tiny-mistral-sample2/iter_0000600 +[rank0]:[2024-08-03 21:31:39,033] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0009948500010068528, 'preprocessing_with_comm': 0.00048758699995232746, 'state_converting': 0.027112548999866704, : 0.02891844899932039}) +------------------------------------------------------------------ +iteration: 2601 , TFLOPS: 19.63792184659827, Tokens per sec: 277406.3827651251, Loss: 8.679828643798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2602 , TFLOPS: 19.237563475060245, Tokens per sec: 271750.8980083539, Loss: 8.681482315063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2603 , TFLOPS: 19.864868043555315, Tokens per sec: 280612.23744119203, Loss: 8.682178497314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2604 , TFLOPS: 19.86213699978176, Tokens per sec: 280573.65856404236, Loss: 8.675981521606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2605 , TFLOPS: 19.85770630122484, Tokens per sec: 280511.07026832557, Loss: 8.680877685546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2606 , TFLOPS: 19.76969966918112, Tokens per sec: 279267.8836600227, Loss: 8.67735481262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2607 , TFLOPS: 19.864151386957822, Tokens per sec: 280602.1139099982, Loss: 8.675887107849121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2608 , TFLOPS: 19.872974210158837, Tokens per sec: 280726.74560418096, Loss: 8.678245544433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2609 , TFLOPS: 19.669867574846673, Tokens per sec: 277857.64990975533, Loss: 8.67768383026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2610 , TFLOPS: 19.69033828902866, Tokens per sec: 278146.8203636457, Loss: 8.682466506958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2611 , TFLOPS: 19.871470440497408, Tokens per sec: 280705.5032698063, Loss: 8.683012008666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2612 , TFLOPS: 19.767198034845322, Tokens per sec: 279232.5454334264, Loss: 8.67963981628418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2613 , TFLOPS: 19.877872429989296, Tokens per sec: 280795.93813156604, Loss: 8.679293632507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2614 , TFLOPS: 19.87030515701318, Tokens per sec: 280689.0424101094, Loss: 8.681066513061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2615 , TFLOPS: 19.810702782868166, Tokens per sec: 279847.0959381284, Loss: 8.676032066345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2616 , TFLOPS: 19.285924244792646, Tokens per sec: 272434.04494742403, Loss: 8.676830291748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2617 , TFLOPS: 19.882960825170066, Tokens per sec: 280867.81708659005, Loss: 8.682609558105469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2618 , TFLOPS: 19.73915252541874, Tokens per sec: 278836.372998094, Loss: 8.6786527633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2619 , TFLOPS: 19.873430270579117, Tokens per sec: 280733.18793919485, Loss: 8.680383682250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2620 , TFLOPS: 19.904797170067734, Tokens per sec: 281176.27851637837, Loss: 8.68087387084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2621 , TFLOPS: 19.91167918877292, Tokens per sec: 281273.49429766316, Loss: 8.678131103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2622 , TFLOPS: 19.876485409803344, Tokens per sec: 280776.34500681417, Loss: 8.677947998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2623 , TFLOPS: 19.901899565643077, Tokens per sec: 281135.34679415205, Loss: 8.680167198181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2624 , TFLOPS: 19.906681269367056, Tokens per sec: 281202.89340849774, Loss: 8.68079662322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2625 , TFLOPS: 19.90093446657976, Tokens per sec: 281121.71375078545, Loss: 8.681782722473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2626 , TFLOPS: 19.903333999851014, Tokens per sec: 281155.6096920312, Loss: 8.678556442260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2627 , TFLOPS: 19.900494134533993, Tokens per sec: 281115.49359567, Loss: 8.680538177490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2628 , TFLOPS: 19.904545148731998, Tokens per sec: 281172.718448889, Loss: 8.679558753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2629 , TFLOPS: 19.901291366332174, Tokens per sec: 281126.7553366568, Loss: 8.67892074584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2630 , TFLOPS: 19.902523951815326, Tokens per sec: 281144.1669081562, Loss: 8.678659439086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2631 , TFLOPS: 19.305735080783247, Tokens per sec: 272713.89392506203, Loss: 8.674223899841309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2632 , TFLOPS: 19.89977147027129, Tokens per sec: 281105.2851998612, Loss: 8.674725532531738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2633 , TFLOPS: 19.89594533166826, Tokens per sec: 281051.23695187876, Loss: 8.677948951721191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2634 , TFLOPS: 19.90210917380611, Tokens per sec: 281138.3077307908, Loss: 8.68099594116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2635 , TFLOPS: 19.897627579163704, Tokens per sec: 281075.00047412276, Loss: 8.682422637939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2636 , TFLOPS: 19.8967147012401, Tokens per sec: 281062.10510949756, Loss: 8.678346633911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2637 , TFLOPS: 19.8930329093721, Tokens per sec: 281010.0959115704, Loss: 8.684160232543945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2638 , TFLOPS: 19.906786392656556, Tokens per sec: 281204.378385967, Loss: 8.677384376525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2639 , TFLOPS: 19.907345091044817, Tokens per sec: 281212.27059065923, Loss: 8.682888984680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2640 , TFLOPS: 19.901873485072752, Tokens per sec: 281134.9783785564, Loss: 8.680335998535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2641 , TFLOPS: 19.900684676424422, Tokens per sec: 281118.1852010751, Loss: 8.678735733032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2642 , TFLOPS: 19.89024522981932, Tokens per sec: 280970.71699423465, Loss: 8.679741859436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2643 , TFLOPS: 19.90599967945115, Tokens per sec: 281193.26523121, Loss: 8.68206787109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2644 , TFLOPS: 19.90978094260853, Tokens per sec: 281246.6795661364, Loss: 8.678536415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2645 , TFLOPS: 19.269183349655314, Tokens per sec: 272197.56212604535, Loss: 8.676565170288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2646 , TFLOPS: 19.907808964529195, Tokens per sec: 281218.82329345366, Loss: 8.685683250427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2647 , TFLOPS: 19.881193188408044, Tokens per sec: 280842.84735078877, Loss: 8.680728912353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2648 , TFLOPS: 19.866881517532033, Tokens per sec: 280640.6798872429, Loss: 8.679122924804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2649 , TFLOPS: 17.379161951435343, Tokens per sec: 245499.01410632668, Loss: 8.676071166992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2650 , TFLOPS: 19.90304269237911, Tokens per sec: 281151.49466638424, Loss: 8.678945541381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2651 , TFLOPS: 19.89477400174049, Tokens per sec: 281034.6906797821, Loss: 8.676878929138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2652 , TFLOPS: 19.895995138696456, Tokens per sec: 281051.9405287459, Loss: 8.678768157958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2653 , TFLOPS: 19.838360628587832, Tokens per sec: 280237.79221424676, Loss: 8.677898406982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2654 , TFLOPS: 19.697165810220067, Tokens per sec: 278243.26631000167, Loss: 8.683161735534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2655 , TFLOPS: 19.796212395106885, Tokens per sec: 279642.4039098618, Loss: 8.681195259094238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2656 , TFLOPS: 19.882024706193064, Tokens per sec: 280854.5934175442, Loss: 8.680014610290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2657 , TFLOPS: 19.88299352385303, Tokens per sec: 280868.2789900151, Loss: 8.677522659301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2658 , TFLOPS: 19.844429107831033, Tokens per sec: 280323.5158915733, Loss: 8.67828369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2659 , TFLOPS: 19.78929458994161, Tokens per sec: 279544.68260703946, Loss: 8.678797721862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2660 , TFLOPS: 19.48833609927912, Tokens per sec: 275293.3261290324, Loss: 8.684615135192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2661 , TFLOPS: 19.875535759289818, Tokens per sec: 280762.9301905263, Loss: 8.679911613464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2662 , TFLOPS: 19.802523041156785, Tokens per sec: 279731.54844904924, Loss: 8.677080154418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2663 , TFLOPS: 19.77576919641057, Tokens per sec: 279353.62214126164, Loss: 8.681793212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2664 , TFLOPS: 19.873376688496144, Tokens per sec: 280732.4310356928, Loss: 8.678814888000488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2665 , TFLOPS: 19.873431330941564, Tokens per sec: 280733.2029179341, Loss: 8.679551124572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2666 , TFLOPS: 19.87472584457766, Tokens per sec: 280751.48928998684, Loss: 8.677545547485352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2667 , TFLOPS: 19.78937716577908, Tokens per sec: 279545.84907793975, Loss: 8.67916202545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2668 , TFLOPS: 19.863453000612182, Tokens per sec: 280592.2484654043, Loss: 8.67652702331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2669 , TFLOPS: 19.855894834263786, Tokens per sec: 280485.4813846805, Loss: 8.679661750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2670 , TFLOPS: 19.723794507849274, Tokens per sec: 278619.42478261236, Loss: 8.681591033935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2671 , TFLOPS: 19.837616896037446, Tokens per sec: 280227.28620662744, Loss: 8.680351257324219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2672 , TFLOPS: 19.886891267436575, Tokens per sec: 280923.33873395866, Loss: 8.679649353027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2673 , TFLOPS: 19.864914245094848, Tokens per sec: 280612.8900867245, Loss: 8.680021286010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2674 , TFLOPS: 19.257425640132837, Tokens per sec: 272031.472063476, Loss: 8.677412986755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2675 , TFLOPS: 19.76462594463735, Tokens per sec: 279196.2119432377, Loss: 8.67874526977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2676 , TFLOPS: 19.860720139627826, Tokens per sec: 280553.64391823317, Loss: 8.680438041687012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2677 , TFLOPS: 19.676313192522766, Tokens per sec: 277948.7010656861, Loss: 8.676375389099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2678 , TFLOPS: 19.854822722361455, Tokens per sec: 280470.33667196246, Loss: 8.678147315979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2679 , TFLOPS: 19.84361606081864, Tokens per sec: 280312.03074398544, Loss: 8.67670726776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2680 , TFLOPS: 19.74471308479507, Tokens per sec: 278914.92177096254, Loss: 8.678994178771973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2681 , TFLOPS: 19.764317439373293, Tokens per sec: 279191.85398061655, Loss: 8.67956256866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2682 , TFLOPS: 19.855591411638013, Tokens per sec: 280481.195219692, Loss: 8.677720069885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2683 , TFLOPS: 19.854227606935968, Tokens per sec: 280461.9300381648, Loss: 8.679803848266602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2684 , TFLOPS: 19.85959635415767, Tokens per sec: 280537.769261805, Loss: 8.679425239562988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2685 , TFLOPS: 19.859547448695892, Tokens per sec: 280537.0784205155, Loss: 8.679500579833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2686 , TFLOPS: 19.860860872947924, Tokens per sec: 280555.63192498713, Loss: 8.678390502929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2687 , TFLOPS: 19.736105078985904, Tokens per sec: 278793.32459927636, Loss: 8.680285453796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2688 , TFLOPS: 19.826567503223, Tokens per sec: 280071.2018655067, Loss: 8.683002471923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2689 , TFLOPS: 19.20825665834361, Tokens per sec: 271336.9082756709, Loss: 8.675983428955078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2690 , TFLOPS: 19.826698355995337, Tokens per sec: 280073.0502990901, Loss: 8.678500175476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2691 , TFLOPS: 19.732814795743234, Tokens per sec: 278746.845874096, Loss: 8.68085765838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2692 , TFLOPS: 19.830574273705594, Tokens per sec: 280127.80172952637, Loss: 8.678788185119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2693 , TFLOPS: 19.809968754439762, Tokens per sec: 279836.7270114792, Loss: 8.682432174682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2694 , TFLOPS: 19.848677235011227, Tokens per sec: 280383.5251738074, Loss: 8.681209564208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2695 , TFLOPS: 19.850769208112524, Tokens per sec: 280413.07650288416, Loss: 8.682848930358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2696 , TFLOPS: 19.64808163645684, Tokens per sec: 277549.90052511694, Loss: 8.68177604675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2697 , TFLOPS: 19.866518202579442, Tokens per sec: 280635.54767989484, Loss: 8.677225112915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2698 , TFLOPS: 19.818058024155697, Tokens per sec: 279950.9964880887, Loss: 8.67934799194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2699 , TFLOPS: 19.85465180281058, Tokens per sec: 280467.92225282366, Loss: 8.677523612976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2700 , TFLOPS: 19.854504522489282, Tokens per sec: 280465.84176276455, Loss: 8.678031921386719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2701 , TFLOPS: 19.836528404934825, Tokens per sec: 280211.91012040974, Loss: 8.67872428894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2702 , TFLOPS: 19.853829716326825, Tokens per sec: 280456.30941315764, Loss: 8.677563667297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2703 , TFLOPS: 19.858262668818405, Tokens per sec: 280518.92954808153, Loss: 8.673417091369629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2704 , TFLOPS: 19.234183408793964, Tokens per sec: 271703.15100315853, Loss: 8.68307113647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2705 , TFLOPS: 19.721518993250303, Tokens per sec: 278587.2807360701, Loss: 8.684283256530762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2706 , TFLOPS: 19.861794803107642, Tokens per sec: 280568.8246746775, Loss: 8.678727149963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2707 , TFLOPS: 19.748941765956264, Tokens per sec: 278974.65635764424, Loss: 8.675100326538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2708 , TFLOPS: 19.857295581061273, Tokens per sec: 280505.26841231476, Loss: 8.677421569824219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2709 , TFLOPS: 19.85636707489031, Tokens per sec: 280492.152282193, Loss: 8.67894172668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2710 , TFLOPS: 19.631732830410698, Tokens per sec: 277318.95637617813, Loss: 8.679956436157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2711 , TFLOPS: 19.85817953066681, Tokens per sec: 280517.75513390044, Loss: 8.678126335144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2712 , TFLOPS: 19.859607872942924, Tokens per sec: 280537.9319768099, Loss: 8.679885864257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2713 , TFLOPS: 19.858030630281203, Tokens per sec: 280515.65175872226, Loss: 8.678085327148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2714 , TFLOPS: 19.75024852877294, Tokens per sec: 278993.11576230725, Loss: 8.677438735961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2715 , TFLOPS: 19.858747333972623, Tokens per sec: 280525.77595516754, Loss: 8.677691459655762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2716 , TFLOPS: 19.858513214071763, Tokens per sec: 280522.46876434877, Loss: 8.67638111114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2717 , TFLOPS: 19.854107835109172, Tokens per sec: 280460.2381346381, Loss: 8.679120063781738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2718 , TFLOPS: 19.263994907569565, Tokens per sec: 272124.26990284387, Loss: 8.674910545349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2719 , TFLOPS: 19.756312571912304, Tokens per sec: 279078.7767749828, Loss: 8.677522659301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2720 , TFLOPS: 19.860996991930286, Tokens per sec: 280557.5547493479, Loss: 8.676981925964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2721 , TFLOPS: 19.864773128359985, Tokens per sec: 280610.89666383225, Loss: 8.67729377746582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2722 , TFLOPS: 19.86382164953151, Tokens per sec: 280597.45602066506, Loss: 8.677741050720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2723 , TFLOPS: 19.72550330147233, Tokens per sec: 278643.56329694024, Loss: 8.678622245788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2724 , TFLOPS: 19.81914882906635, Tokens per sec: 279966.4052592881, Loss: 8.67975902557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2725 , TFLOPS: 19.859895887526804, Tokens per sec: 280542.0004869365, Loss: 8.678889274597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2726 , TFLOPS: 19.804993061598633, Tokens per sec: 279766.44009601075, Loss: 8.678908348083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2727 , TFLOPS: 19.803893352244117, Tokens per sec: 279750.90554013907, Loss: 8.676592826843262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2728 , TFLOPS: 19.778433647076607, Tokens per sec: 279391.26032044855, Loss: 8.680198669433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2729 , TFLOPS: 19.723872828776393, Tokens per sec: 278620.531148411, Loss: 8.68066120147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2730 , TFLOPS: 19.860005065225565, Tokens per sec: 280543.5427372178, Loss: 8.680395126342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2731 , TFLOPS: 19.85744124016331, Tokens per sec: 280507.32600094, Loss: 8.679489135742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2732 , TFLOPS: 19.861846252023028, Tokens per sec: 280569.55144493486, Loss: 8.6782865524292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2733 , TFLOPS: 19.19194118840855, Tokens per sec: 271106.4350345024, Loss: 8.680286407470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2734 , TFLOPS: 19.899067953379483, Tokens per sec: 281095.34728088515, Loss: 8.684672355651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2735 , TFLOPS: 19.856232600354446, Tokens per sec: 280490.25268737547, Loss: 8.68089771270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2736 , TFLOPS: 19.901058546098003, Tokens per sec: 281123.46650500246, Loss: 8.675503730773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2737 , TFLOPS: 19.905215469612916, Tokens per sec: 281182.1874391579, Loss: 8.679427146911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2738 , TFLOPS: 15.410860953658428, Tokens per sec: 217694.6840834499, Loss: 8.67689323425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2739 , TFLOPS: 20.070524878550103, Tokens per sec: 283517.3573990205, Loss: 8.680435180664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2740 , TFLOPS: 20.075592427126058, Tokens per sec: 283588.94187373854, Loss: 8.675973892211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2741 , TFLOPS: 19.999441229042397, Tokens per sec: 282513.22578887804, Loss: 8.677863121032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2742 , TFLOPS: 19.933459754044783, Tokens per sec: 281581.1677813365, Loss: 8.68027400970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2743 , TFLOPS: 19.98785097512377, Tokens per sec: 282349.50121353794, Loss: 8.678617477416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2744 , TFLOPS: 20.02458481214611, Tokens per sec: 282868.40545060806, Loss: 8.679781913757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2745 , TFLOPS: 19.879630981300405, Tokens per sec: 280820.77952577936, Loss: 8.678622245788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2746 , TFLOPS: 19.89247252147951, Tokens per sec: 281002.1798408456, Loss: 8.681133270263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2747 , TFLOPS: 20.02421767306735, Tokens per sec: 282863.21922343975, Loss: 8.67006778717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2748 , TFLOPS: 20.026289402085595, Tokens per sec: 282892.4845834666, Loss: 8.678335189819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2749 , TFLOPS: 19.924394204493506, Tokens per sec: 281453.10732114926, Loss: 8.678145408630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2750 , TFLOPS: 19.616310217214163, Tokens per sec: 277101.0956792543, Loss: 8.678499221801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2751 , TFLOPS: 20.033655312825736, Tokens per sec: 282996.53585071146, Loss: 8.68104076385498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2752 , TFLOPS: 19.980550918564024, Tokens per sec: 282246.38020613004, Loss: 8.679365158081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2753 , TFLOPS: 20.054527153874776, Tokens per sec: 283291.3726451692, Loss: 8.67674732208252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2754 , TFLOPS: 20.053927336379836, Tokens per sec: 283282.8995896753, Loss: 8.678421974182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2755 , TFLOPS: 19.907092984727807, Tokens per sec: 281208.7093227196, Loss: 8.677924156188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2756 , TFLOPS: 20.051836606348523, Tokens per sec: 283253.3658202747, Loss: 8.678173065185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2757 , TFLOPS: 19.92324152615085, Tokens per sec: 281436.8245223874, Loss: 8.677497863769531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2758 , TFLOPS: 20.05189754898714, Tokens per sec: 283254.22669939755, Loss: 8.678382873535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2759 , TFLOPS: 20.049639618292836, Tokens per sec: 283222.3310440765, Loss: 8.679913520812988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2760 , TFLOPS: 19.935744983403385, Tokens per sec: 281613.4490591169, Loss: 8.675939559936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2761 , TFLOPS: 19.769485273735086, Tokens per sec: 279264.8550979564, Loss: 8.682259559631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2762 , TFLOPS: 20.036476975524916, Tokens per sec: 283036.3948158747, Loss: 8.678650856018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2763 , TFLOPS: 20.009866474730966, Tokens per sec: 282660.493392777, Loss: 8.678908348083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2764 , TFLOPS: 19.11695572730694, Tokens per sec: 270047.1862154745, Loss: 8.679513931274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2765 , TFLOPS: 19.98550224316918, Tokens per sec: 282316.3228945337, Loss: 8.680474281311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2766 , TFLOPS: 19.900377811421247, Tokens per sec: 281113.85040887084, Loss: 8.674161911010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2767 , TFLOPS: 19.881131517136666, Tokens per sec: 280841.9761789587, Loss: 8.680546760559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2768 , TFLOPS: 19.978784435560886, Tokens per sec: 282221.4267683913, Loss: 8.675670623779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2769 , TFLOPS: 18.5831129586416, Tokens per sec: 262506.09339630615, Loss: 8.675949096679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2770 , TFLOPS: 19.873478150083244, Tokens per sec: 280733.86428774905, Loss: 8.68075942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2771 , TFLOPS: 19.697805515212714, Tokens per sec: 278252.3028185199, Loss: 8.678226470947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2772 , TFLOPS: 19.9404855183059, Tokens per sec: 281680.41412038717, Loss: 8.678967475891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2773 , TFLOPS: 19.952325136530366, Tokens per sec: 281847.66122986365, Loss: 8.680173873901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2774 , TFLOPS: 19.644545041022237, Tokens per sec: 277499.9423801319, Loss: 8.680981636047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2775 , TFLOPS: 19.913963385061635, Tokens per sec: 281305.76098224055, Loss: 8.675192832946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2776 , TFLOPS: 19.92071454451893, Tokens per sec: 281401.12823846797, Loss: 8.682313919067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2777 , TFLOPS: 19.909605978361217, Tokens per sec: 281244.2080113895, Loss: 8.676979064941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2778 , TFLOPS: 19.91315623868371, Tokens per sec: 281294.35918734537, Loss: 8.67848014831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2779 , TFLOPS: 19.01986908431715, Tokens per sec: 268675.73486451234, Loss: 8.678584098815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2780 , TFLOPS: 19.776895375232403, Tokens per sec: 279369.5306063096, Loss: 8.677578926086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2781 , TFLOPS: 19.895086845181105, Tokens per sec: 281039.10992372816, Loss: 8.678175926208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2782 , TFLOPS: 19.78077263124172, Tokens per sec: 279424.30094163335, Loss: 8.674934387207031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2783 , TFLOPS: 19.96045938573306, Tokens per sec: 281962.56608921976, Loss: 8.680375099182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2784 , TFLOPS: 19.96677610394829, Tokens per sec: 282051.7964041544, Loss: 8.677635192871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2785 , TFLOPS: 19.88759434560935, Tokens per sec: 280933.2704555652, Loss: 8.676702499389648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2786 , TFLOPS: 19.90825324915433, Tokens per sec: 281225.09928292653, Loss: 8.679564476013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2787 , TFLOPS: 19.934440286350974, Tokens per sec: 281595.0188355555, Loss: 8.677464485168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2788 , TFLOPS: 19.758282229642283, Tokens per sec: 279106.6002702806, Loss: 8.674806594848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2789 , TFLOPS: 19.85067431230917, Tokens per sec: 280411.73599945643, Loss: 8.680767059326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2790 , TFLOPS: 19.816796084138865, Tokens per sec: 279933.1702528042, Loss: 8.677240371704102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2791 , TFLOPS: 19.802071376531938, Tokens per sec: 279725.1682092876, Loss: 8.67686939239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2792 , TFLOPS: 19.71880727180865, Tokens per sec: 278548.97480726195, Loss: 8.685070991516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2793 , TFLOPS: 19.91351619076999, Tokens per sec: 281299.4438906558, Loss: 8.672930717468262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2794 , TFLOPS: 19.3422291256221, Tokens per sec: 273229.4108443316, Loss: 8.6829195022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2795 , TFLOPS: 19.780891035051244, Tokens per sec: 279425.9735204714, Loss: 8.675300598144531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2796 , TFLOPS: 19.42506172612495, Tokens per sec: 274399.5087935963, Loss: 8.674861907958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2797 , TFLOPS: 19.90722456836979, Tokens per sec: 281210.56808060844, Loss: 8.67692756652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2798 , TFLOPS: 19.91807879516145, Tokens per sec: 281363.89549557155, Loss: 8.676210403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2799 , TFLOPS: 19.768364269859624, Tokens per sec: 279249.0197344904, Loss: 8.678833961486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2800 , TFLOPS: 19.5323929029219, Tokens per sec: 275915.67500230955, Loss: 8.678823471069336 +------------------------------------------------------------------ +[rank0]:[2024-08-03 21:41:28,164] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0020187720001558773, 'preprocessing_with_comm': 0.0007257450015458744, 'state_converting': 0.028989177000767086, : 0.032050874000560725}) + eval ppl=5890.47216796875, eval loss=8.68109130859375 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample2/iter_0002800, took 0.20s +Deleting checkpoint /work/llm_recipes/models/tiny-mistral-sample2/iter_0000800 +------------------------------------------------------------------ +iteration: 2801 , TFLOPS: 19.883478668898448, Tokens per sec: 280875.132176071, Loss: 8.677680969238281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2802 , TFLOPS: 19.90154822673485, Tokens per sec: 281130.38376107666, Loss: 8.676876068115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2803 , TFLOPS: 19.89411352473644, Tokens per sec: 281025.36074466864, Loss: 8.674016952514648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2804 , TFLOPS: 19.906113531408025, Tokens per sec: 281194.873510323, Loss: 8.677329063415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2805 , TFLOPS: 19.887060026705107, Tokens per sec: 280925.7226368282, Loss: 8.675642967224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2806 , TFLOPS: 19.880339402649295, Tokens per sec: 280830.78672539064, Loss: 8.676196098327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2807 , TFLOPS: 19.887078397516525, Tokens per sec: 280925.98214393813, Loss: 8.680204391479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2808 , TFLOPS: 19.93819616439589, Tokens per sec: 281648.0745789628, Loss: 8.681106567382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2809 , TFLOPS: 19.6192448155507, Tokens per sec: 277142.5499795497, Loss: 8.676072120666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2810 , TFLOPS: 19.93477181851536, Tokens per sec: 281599.7020774592, Loss: 8.673622131347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2811 , TFLOPS: 19.865846152137202, Tokens per sec: 280626.0542577458, Loss: 8.679097175598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2812 , TFLOPS: 19.939970024664778, Tokens per sec: 281673.1322283703, Loss: 8.679372787475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2813 , TFLOPS: 19.931547767484624, Tokens per sec: 281554.1589522103, Loss: 8.67821979522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2814 , TFLOPS: 19.845415710608535, Tokens per sec: 280337.4526976096, Loss: 8.680004119873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2815 , TFLOPS: 19.922298106341707, Tokens per sec: 281423.4977213783, Loss: 8.673431396484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2816 , TFLOPS: 19.936496557277863, Tokens per sec: 281624.06583371904, Loss: 8.677199363708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2817 , TFLOPS: 19.86183213715381, Tokens per sec: 280569.3520575026, Loss: 8.677103996276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2818 , TFLOPS: 19.79266023500733, Tokens per sec: 279592.22589754884, Loss: 8.68052864074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2819 , TFLOPS: 19.915416807997275, Tokens per sec: 281326.29211594973, Loss: 8.678823471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2820 , TFLOPS: 19.931978172818702, Tokens per sec: 281560.2388820411, Loss: 8.677528381347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2821 , TFLOPS: 19.929348300976688, Tokens per sec: 281523.089165257, Loss: 8.679075241088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2822 , TFLOPS: 19.824604594419277, Tokens per sec: 280043.47370591876, Loss: 8.67363452911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2823 , TFLOPS: 19.9322111085393, Tokens per sec: 281563.5293450632, Loss: 8.674670219421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2824 , TFLOPS: 19.264890657868296, Tokens per sec: 272136.9233216818, Loss: 8.675214767456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2825 , TFLOPS: 19.655473795594613, Tokens per sec: 277654.3225786953, Loss: 8.675392150878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2826 , TFLOPS: 19.893806031406992, Tokens per sec: 281021.0170766923, Loss: 8.674504280090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2827 , TFLOPS: 19.88909014740536, Tokens per sec: 280954.4002354271, Loss: 8.678186416625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2828 , TFLOPS: 19.898159556447013, Tokens per sec: 281082.5152149914, Loss: 8.676288604736328 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240803_192355-n3hnzq4n/files/requirements.txt b/wandb/run-20240803_192355-n3hnzq4n/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240803_192355-n3hnzq4n/files/wandb-metadata.json b/wandb/run-20240803_192355-n3hnzq4n/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c2e57b81180a1166fe6981d41acf594e2c000f52 --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-03T10:23:55.999895", + "startedAt": "2024-08-03T10:23:55.358106", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "40", + "--global-batch-size", + "1600", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/custom/tiny-mistral", + "--save", + "/work/llm_recipes/models/tiny-mistral-sample2", + "--load", + "/work/llm_recipes/models/tiny-mistral-sample2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-mistral-sample2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-mistral-sample2_train_2024-08-03-19:23:42" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json b/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6ce85cad06b83fd77cde0821ab707eaacd5796b5 --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 8.676288604736328, "training/perplexity": 5862.24904336478, "utils/batch_size": 40, "utils/global_batch_size": 1600, "utils/seq_len": 513, "utils/gradient_accumulation_steps": 40, "utils/iteration": 2828, "optimizer/lr": 1.933962231874466e-05, "optimizer/variance_l2": 0.014160344368509084, "optimizer/variance_sqrt_l2": 0.9983195251588314, "optimizer/momentum_l2": 0.9847836932741917, "optimizer/weight_l2": 101.93656115447489, "optimizer/variance_l1": 1.0000762939453125, "optimizer/variance_sqrt_l1": 530.25, "optimizer/momentum_l1": 418.75, "optimizer/weight_l1": 333248.0, "optimizer/variance_abs_max": 0.00130462646484375, "optimizer/variance_sqrt_abs_max": 0.0361328125, "optimizer/momentum_abs_max": 0.03662109375, "optimizer/weight_abs_max": 1.0, "stats/1_iteration_time": 2.9201389469999413, "stats/tokens_per_sec": 281082.5152149914, "stats/tokens_per_sec_per_gpu": 281082.5152149914, "stats/tflops": 19.898159556447013, "_timestamp": 1722688970.2608888, "_runtime": 8334.889575719833, "_step": 2828, "evaluation/val_loss": 8.68109130859375, "evaluation/val_ppl": 5890.47216796875, "_wandb": {"runtime": 8336}} \ No newline at end of file diff --git a/wandb/run-20240803_192355-n3hnzq4n/logs/debug-internal.log b/wandb/run-20240803_192355-n3hnzq4n/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..21a41e9f4bb226fdf589fc21b7656ae5d4f5de7e --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/logs/debug-internal.log @@ -0,0 +1,20547 @@ +2024-08-03 19:23:55,371 INFO StreamThr :10151 [internal.py:wandb_internal():86] W&B internal server running at pid: 10151, started at: 2024-08-03 19:23:55.370937 +2024-08-03 19:23:55,373 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status +2024-08-03 19:23:55,376 INFO WriterThread:10151 [datastore.py:open_for_write():87] open: /project/wandb/run-20240803_192355-n3hnzq4n/run-n3hnzq4n.wandb +2024-08-03 19:23:55,377 DEBUG SenderThread:10151 [sender.py:send():382] send: header +2024-08-03 19:23:55,407 DEBUG SenderThread:10151 [sender.py:send():382] send: run +2024-08-03 19:23:55,883 INFO SenderThread:10151 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240803_192355-n3hnzq4n/files +2024-08-03 19:23:55,883 INFO SenderThread:10151 [sender.py:_start_run_threads():1136] run started: n3hnzq4n with start time 1722680635.371313 +2024-08-03 19:23:55,888 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: check_version +2024-08-03 19:23:55,889 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: check_version +2024-08-03 19:23:55,980 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: run_start +2024-08-03 19:23:55,986 DEBUG HandlerThread:10151 [system_info.py:__init__():27] System info init +2024-08-03 19:23:55,986 DEBUG HandlerThread:10151 [system_info.py:__init__():42] System info init done +2024-08-03 19:23:55,986 INFO HandlerThread:10151 [system_monitor.py:start():194] Starting system monitor +2024-08-03 19:23:55,986 INFO SystemMonitor:10151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-03 19:23:55,987 INFO HandlerThread:10151 [system_monitor.py:probe():214] Collecting system info +2024-08-03 19:23:55,987 INFO SystemMonitor:10151 [interfaces.py:start():190] Started cpu monitoring +2024-08-03 19:23:55,988 INFO SystemMonitor:10151 [interfaces.py:start():190] Started disk monitoring +2024-08-03 19:23:55,989 INFO SystemMonitor:10151 [interfaces.py:start():190] Started gpu monitoring +2024-08-03 19:23:55,989 INFO SystemMonitor:10151 [interfaces.py:start():190] Started memory monitoring +2024-08-03 19:23:55,990 INFO SystemMonitor:10151 [interfaces.py:start():190] Started network monitoring +2024-08-03 19:23:55,999 DEBUG HandlerThread:10151 [system_info.py:probe():151] Probing system +2024-08-03 19:23:56,001 DEBUG HandlerThread:10151 [system_info.py:_probe_git():136] Probing git +2024-08-03 19:23:56,013 DEBUG HandlerThread:10151 [system_info.py:_probe_git():144] Probing git done +2024-08-03 19:23:56,013 DEBUG HandlerThread:10151 [system_info.py:probe():199] Probing system done +2024-08-03 19:23:56,013 DEBUG HandlerThread:10151 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-03T10:23:55.999895', 'startedAt': '2024-08-03T10:23:55.358106', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '40', '--global-batch-size', '1600', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample2', '--load', '/work/llm_recipes/models/tiny-mistral-sample2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample2_train_2024-08-03-19:23:42'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.034, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-03 19:23:56,013 INFO HandlerThread:10151 [system_monitor.py:probe():224] Finished collecting system info +2024-08-03 19:23:56,013 INFO HandlerThread:10151 [system_monitor.py:probe():227] Publishing system info +2024-08-03 19:23:56,014 INFO HandlerThread:10151 [system_monitor.py:probe():229] Finished publishing system info +2024-08-03 19:23:56,025 DEBUG SenderThread:10151 [sender.py:send():382] send: files +2024-08-03 19:23:56,026 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-03 19:23:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: python_packages +2024-08-03 19:23:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:23:56,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: python_packages +2024-08-03 19:23:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:23:56,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:23:56,355 DEBUG SenderThread:10151 [sender.py:send():382] send: telemetry +2024-08-03 19:23:56,696 INFO wandb-upload_0:10151 [upload_job.py:push():131] Uploaded file /tmp/tmpt920zsgzwandb/9i7atmsi-wandb-metadata.json +2024-08-03 19:23:56,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:23:56,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_192355-n3hnzq4n/files/requirements.txt +2024-08-03 19:23:56,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-metadata.json +2024-08-03 19:23:57,142 DEBUG SenderThread:10151 [sender.py:send():382] send: config +2024-08-03 19:23:57,142 DEBUG SenderThread:10151 [sender.py:send():382] send: config +2024-08-03 19:23:58,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:01,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:02,783 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:04,888 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:05,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:05,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:05,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:05,705 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:05,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:06,705 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:06,890 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:08,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:08,616 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:08,617 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:08,618 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:08,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:10,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:24:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:24:11,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:24:11,526 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:11,528 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:11,528 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:11,529 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:11,893 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:12,569 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:12,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:14,458 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:14,461 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:14,461 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:14,462 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:14,895 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:16,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:17,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:17,389 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:17,390 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:17,391 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:17,897 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:18,392 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:18,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:20,297 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:20,299 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:20,300 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:20,301 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:20,899 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:22,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:23,206 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:23,209 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:23,209 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:23,210 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:23,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:24,211 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:24,902 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:26,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:24:26,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:24:26,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:24:26,116 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:26,200 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:26,201 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:26,202 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:26,903 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:28,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:29,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:29,053 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:29,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:29,054 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:29,905 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:30,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:30,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:30,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/config.yaml +2024-08-03 19:24:31,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:31,961 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:31,962 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:31,963 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:32,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:34,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:34,872 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:34,872 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:34,873 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:34,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:34,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:35,874 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:36,910 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:37,778 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:37,780 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:37,781 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:37,782 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:37,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:38,912 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:40,687 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:40,690 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:40,690 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:40,692 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:40,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:41,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:24:41,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:24:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:24:41,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:42,914 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:43,597 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:43,598 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:43,598 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:43,600 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:43,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:44,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:46,597 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:46,600 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:46,600 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:46,600 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:46,601 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:46,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:48,918 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:49,508 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:49,510 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:49,510 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:49,511 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:49,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:50,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:52,497 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:52,500 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:52,500 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:52,500 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:52,502 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:52,921 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:54,922 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:55,411 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:55,413 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:55,413 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:55,414 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:55,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:24:55,990 DEBUG SystemMonitor:10151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-03 19:24:55,991 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:24:56,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:24:56,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:24:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:24:56,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:24:58,280 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:24:58,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:24:58,332 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:24:58,333 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:24:58,334 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:24:58,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:00,926 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:01,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:01,242 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:01,242 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:01,243 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:01,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:02,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:04,148 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:04,150 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:04,151 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:04,151 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:04,152 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:04,929 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:06,930 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:07,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:07,062 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:07,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:07,063 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:07,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:08,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:09,976 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:09,978 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:09,979 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:09,979 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:09,980 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:10,933 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:11,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:25:11,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:25:11,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:25:12,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:12,890 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:12,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:12,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:12,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:12,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:14,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:15,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:15,843 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:15,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:15,844 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:15,845 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:15,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:16,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:18,766 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:18,768 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:18,769 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:18,770 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:18,938 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:20,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:21,683 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:21,686 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:21,686 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:21,686 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:21,687 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:21,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:22,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:24,603 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:24,605 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:24,606 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:24,607 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:24,942 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:25,993 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:25:26,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:25:26,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:25:26,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:25:26,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:27,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:27,527 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:27,529 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:27,529 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:27,530 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:27,944 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:28,945 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:30,451 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:30,453 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:30,453 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:30,455 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:30,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:32,455 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:32,947 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:33,460 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:33,462 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:33,463 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:33,464 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:33,948 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:34,949 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:36,382 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:36,384 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:36,385 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:36,386 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:36,950 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:38,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:38,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:39,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:39,306 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:39,307 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:39,308 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:39,952 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:40,953 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:41,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:25:41,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:25:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:25:42,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:42,280 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:42,281 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:42,282 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:42,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:44,297 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:44,955 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:45,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:45,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:45,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:45,217 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:45,956 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:46,957 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:48,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:48,139 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:48,139 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:48,140 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:48,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:50,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:50,959 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:51,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:51,069 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:51,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:51,070 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:51,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:52,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:53,994 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:53,997 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:53,998 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:53,999 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:54,962 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:54,962 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:55,993 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:25:55,994 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:25:56,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:25:56,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:25:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:25:56,917 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:56,918 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:56,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:56,920 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:56,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:25:58,964 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:25:59,887 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:25:59,889 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:25:59,889 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:25:59,891 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:25:59,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:00,966 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:01,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:02,825 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:02,830 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:02,830 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:02,831 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:02,967 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:04,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:05,765 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:05,767 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:05,767 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:05,768 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:05,969 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:06,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:07,769 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:08,687 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:08,690 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:08,690 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:08,692 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:08,971 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:10,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:26:11,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:26:11,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:26:11,610 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:11,611 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:11,612 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:11,613 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:11,973 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:12,974 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:13,653 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:14,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:14,624 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:14,624 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:14,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:14,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:16,976 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:17,548 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:17,551 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:17,552 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:17,553 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:17,977 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:18,978 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:19,553 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:20,518 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:20,521 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:20,521 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:20,522 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:20,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:22,980 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:23,465 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:23,467 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:23,468 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:23,469 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:23,981 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:24,982 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:25,469 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:25,994 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:26:26,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:26:26,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:26:26,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:26:26,433 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:26,435 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:26,435 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:26,436 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:26,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:28,984 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:29,383 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:29,385 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:29,386 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:29,387 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:29,985 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:30,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:31,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:32,334 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:32,336 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:32,337 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:32,338 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:32,987 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:34,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:35,269 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:35,271 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:35,271 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:35,272 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:35,989 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:36,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:37,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:38,200 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:38,202 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:38,202 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:38,203 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:38,991 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:40,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:26:41,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:26:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:26:41,134 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:41,206 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:41,207 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:41,208 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:41,993 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:42,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:43,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:44,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:44,067 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:44,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:44,068 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:44,995 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:46,994 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:46,996 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:46,997 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:46,998 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:46,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:46,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:48,998 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:48,999 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:49,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:49,939 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:49,940 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:49,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:50,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:51,001 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:52,861 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:52,863 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:52,863 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:52,864 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:53,002 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:54,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:26:55,003 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:55,816 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:55,818 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:55,819 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:55,820 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:55,995 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:26:56,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:26:56,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:26:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:26:57,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:26:58,885 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:26:58,887 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:26:58,887 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:26:58,888 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:26:59,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:26:59,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:01,007 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:01,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:01,823 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:01,824 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:01,825 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:02,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:03,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:04,745 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:04,747 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:04,748 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:04,749 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:05,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:05,750 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:07,011 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:07,669 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:07,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:07,673 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:07,674 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:08,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:09,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:10,633 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:10,635 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:10,635 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:10,637 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:11,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:27:11,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:27:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:27:11,232 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:13,015 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:13,580 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:13,581 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:13,582 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:13,583 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:14,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:15,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:16,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:16,506 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:16,507 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:16,507 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:16,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:17,018 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:19,019 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:19,438 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:19,441 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:19,441 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:19,442 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:20,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:21,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:22,373 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:22,376 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:22,376 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:22,377 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:22,378 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:23,022 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:25,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:25,307 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:25,310 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:25,310 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:25,311 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:25,995 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:27:26,024 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:26,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:27:26,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:27:26,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:27:27,024 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:28,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:28,241 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:28,243 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:28,243 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:28,244 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:29,026 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:31,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:31,175 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:31,178 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:31,178 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:31,179 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:32,028 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:33,028 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:34,109 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:34,112 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:34,112 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:34,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:34,113 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:35,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:35,030 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:37,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:37,047 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:37,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:37,048 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:38,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:39,032 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:39,979 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:39,981 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:39,981 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:39,982 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:39,983 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:40,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:41,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:27:41,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:27:41,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:27:43,006 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:43,007 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:43,008 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:43,009 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:43,035 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:45,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:45,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:45,956 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:45,958 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:45,958 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:45,959 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:46,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:47,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:48,890 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:48,892 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:48,893 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:48,894 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:49,039 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:50,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:51,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:51,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:51,854 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:51,855 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:51,856 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:52,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:53,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:54,845 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:54,848 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:54,848 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:54,850 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:55,043 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:55,996 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:27:55,997 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:27:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:27:56,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:27:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:27:57,044 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:27:57,792 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:27:57,795 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:27:57,795 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:27:57,796 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:27:58,045 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:27:59,045 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:00,727 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:00,730 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:00,731 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:00,732 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:01,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:01,732 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:03,048 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:03,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:03,663 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:03,664 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:03,665 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:04,049 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:05,049 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:06,613 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:06,615 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:06,616 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:06,617 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:07,051 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:07,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:09,052 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:09,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:09,550 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:09,550 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:09,551 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:10,053 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:28:11,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:28:11,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:28:11,053 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:12,482 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:12,484 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:12,484 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:12,485 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:13,054 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:13,526 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:15,055 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:15,411 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:15,413 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:15,414 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:15,415 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:16,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:17,057 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:18,338 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:18,341 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:18,341 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:18,342 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:19,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:19,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:21,059 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:21,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:21,269 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:21,269 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:21,270 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:22,060 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:23,061 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:24,196 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:24,198 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:24,198 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:24,199 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:25,062 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:25,200 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:25,997 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:28:26,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:28:26,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:28:26,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:28:27,064 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:27,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:27,262 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:27,262 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:27,263 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:28,065 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:29,065 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:30,188 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:30,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:30,191 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:30,192 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:31,067 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:31,067 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:31,193 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:33,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:33,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:33,119 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:33,120 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:34,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:35,069 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:36,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:36,047 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:36,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:36,048 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:36,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:37,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:37,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:38,993 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:38,995 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:38,996 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:38,997 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:39,072 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:28:41,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:28:41,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:28:41,073 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:41,922 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:41,923 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:41,924 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:41,925 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:42,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:42,965 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:43,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:44,851 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:44,855 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:44,856 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:44,857 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:45,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:47,077 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:47,779 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:47,782 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:47,782 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:47,783 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:48,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:48,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:49,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:50,707 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:50,710 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:50,710 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:50,711 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:51,080 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:53,081 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:53,660 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:53,662 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:53,662 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:53,663 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:54,082 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:54,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:28:55,082 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:55,998 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:28:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:28:56,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:28:56,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:28:56,588 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:56,589 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:56,590 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:56,591 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:28:57,084 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:28:59,085 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:28:59,531 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:28:59,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:28:59,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:28:59,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:00,086 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:00,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:01,086 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:02,461 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:02,463 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:02,464 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:02,465 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:03,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:05,089 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:05,415 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:05,417 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:05,418 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:05,419 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:06,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:06,419 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:07,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:08,344 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:08,346 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:08,347 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:08,348 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:09,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:11,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:29:11,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:29:11,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:29:11,093 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:11,370 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:11,371 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:11,371 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:11,372 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:12,094 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:12,413 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:13,094 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:14,314 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:14,316 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:14,316 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:14,317 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:15,096 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:17,097 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:17,241 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:17,244 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:17,244 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:17,245 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:18,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:18,245 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:19,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:20,171 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:20,173 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:20,174 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:20,175 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:21,100 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:21,100 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:23,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:23,103 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:23,103 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:23,104 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:24,104 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:24,104 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:25,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:25,999 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:29:26,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:29:26,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:29:26,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:29:26,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:26,269 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:26,269 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:26,270 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:27,106 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:28,995 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:28,998 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:28,998 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:28,999 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:29,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:29,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:30,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:31,109 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:31,949 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:31,952 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:31,952 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:31,953 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:32,110 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:33,110 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:34,878 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:34,880 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:34,881 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:34,882 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:35,112 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:35,882 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:37,113 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:37,833 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:37,835 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:37,836 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:37,837 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:38,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:39,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:40,781 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:40,783 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:40,784 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:40,785 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:41,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:29:41,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:29:41,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:29:41,116 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:41,254 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:43,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:43,711 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:43,712 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:43,713 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:43,714 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:44,118 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:45,118 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:46,657 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:46,659 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:46,659 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:46,659 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:46,660 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:47,120 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:49,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:49,587 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:49,589 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:49,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:49,590 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:50,122 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:51,122 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:52,591 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:52,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:52,608 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:52,609 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:52,610 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:53,124 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:55,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:55,537 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:55,539 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:55,539 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:55,540 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:56,000 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:29:56,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:29:56,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:29:56,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:29:56,126 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:29:57,126 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:29:58,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:29:58,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:29:58,467 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:29:58,468 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:29:58,469 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:29:59,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:01,129 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:01,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:01,397 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:01,397 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:01,398 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:02,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:03,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:03,399 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:04,323 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:04,326 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:04,326 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:04,327 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:05,132 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:07,133 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:07,251 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:07,254 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:07,254 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:07,255 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:08,134 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:09,134 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:09,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:10,180 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:10,183 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:10,183 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:10,184 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:11,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:30:11,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:30:11,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:30:11,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:11,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:13,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:13,108 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:13,108 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:13,109 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:13,137 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:15,138 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:15,149 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:16,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:16,038 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:16,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:16,039 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:16,139 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:17,139 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:18,965 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:18,967 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:18,968 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:18,969 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:19,141 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:20,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:21,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:21,897 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:21,900 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:21,900 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:21,901 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:22,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:23,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:24,827 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:24,829 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:24,829 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:24,830 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:25,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:26,001 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:30:26,001 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:26,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:30:26,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:30:26,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:30:27,146 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:27,776 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:27,778 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:27,778 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:27,779 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:28,147 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:29,148 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:30,710 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:30,712 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:30,713 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:30,714 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:31,149 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:31,714 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:33,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:33,641 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:33,644 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:33,644 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:33,645 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:34,151 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:35,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:36,665 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:36,668 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:36,669 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:36,670 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:37,153 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:37,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:39,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:39,640 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:39,642 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:39,643 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:39,644 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:40,155 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:41,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:30:41,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:30:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:30:41,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:42,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:42,576 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:42,576 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:42,577 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:43,157 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:43,617 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:45,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:45,510 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:45,512 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:45,513 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:45,514 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:46,159 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:47,160 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:48,444 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:48,447 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:48,447 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:48,448 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:49,161 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:49,449 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:51,162 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:51,381 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:51,383 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:51,383 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:51,384 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:52,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:53,164 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:54,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:54,321 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:54,321 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:54,322 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:55,165 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:30:55,165 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:55,323 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:30:56,001 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:30:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:30:56,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:30:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:30:57,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:30:57,272 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:30:57,272 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:30:57,273 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:30:58,167 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:30:59,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:00,228 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:00,230 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:00,230 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:00,231 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:01,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:01,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:01,232 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:03,178 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:03,180 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:03,180 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:03,181 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:04,171 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:05,172 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:06,169 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:06,171 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:06,171 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:06,173 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:06,173 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:07,173 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:07,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:09,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:09,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:09,169 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:09,170 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:09,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:11,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:31:11,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:31:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:31:11,176 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:12,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:12,123 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:12,123 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:12,124 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:12,177 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:13,165 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:13,178 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:15,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:15,075 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:15,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:15,077 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:15,179 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:17,180 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:18,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:18,087 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:18,087 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:18,088 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:18,181 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:19,089 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:19,182 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:21,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:21,047 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:21,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:21,049 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:21,183 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:23,184 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:24,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:24,002 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:24,003 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:24,004 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:24,185 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:25,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:25,185 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:26,002 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:31:26,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:31:26,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:31:26,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:31:26,938 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:26,942 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:26,942 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:26,943 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:27,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:29,188 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:29,871 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:29,873 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:29,873 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:29,874 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:30,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:30,875 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:31,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:32,816 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:32,818 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:32,819 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:32,820 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:33,191 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:35,192 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:35,772 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:35,774 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:35,774 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:35,776 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:36,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:36,776 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:37,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:38,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:38,711 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:38,712 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:38,713 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:39,195 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:41,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:31:41,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:31:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:31:41,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:41,638 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:41,640 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:41,640 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:41,641 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:42,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:42,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:43,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:44,573 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:44,575 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:44,576 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:44,577 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:45,198 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:47,199 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:47,510 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:47,512 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:47,512 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:47,514 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:48,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:48,514 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:49,201 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:50,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:50,492 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:50,492 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:50,493 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:51,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:53,203 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:53,403 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:53,405 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:53,405 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:53,406 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:54,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:54,407 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:31:55,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:56,003 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:31:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:31:56,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:31:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:31:56,360 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:56,361 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:56,361 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:56,362 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:31:57,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:31:57,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:31:59,306 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:31:59,309 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:31:59,309 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:31:59,310 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:00,208 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:00,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:01,209 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:02,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:02,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:02,293 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:02,294 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:03,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:03,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:05,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:05,273 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:05,274 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:05,275 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:06,212 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:06,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:07,212 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:08,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:08,210 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:08,211 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:08,212 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:08,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:09,214 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:11,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:32:11,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:32:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:32:11,179 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:11,303 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:11,303 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:11,303 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:11,305 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:12,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:13,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:14,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:14,123 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:14,124 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:14,125 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:14,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:15,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:17,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:17,073 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:17,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:17,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:17,074 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:17,219 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:19,220 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:20,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:20,018 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:20,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:20,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:20,221 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:21,221 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:22,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:22,957 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:22,957 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:22,958 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:22,961 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:23,223 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:25,223 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:25,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:25,893 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:25,894 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:25,895 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:26,004 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:32:26,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:32:26,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:32:26,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:32:26,224 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:27,225 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:28,207 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:28,836 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:28,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:28,838 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:28,839 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:29,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:31,227 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:31,773 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:31,775 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:31,775 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:31,776 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:32,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:33,229 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:33,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:34,747 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:34,750 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:34,750 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:34,751 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:35,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:37,231 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:37,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:37,692 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:37,692 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:37,693 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:38,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:39,233 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:39,694 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:40,633 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:40,636 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:40,636 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:40,637 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:41,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:32:41,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:32:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:32:41,234 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:43,235 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:43,587 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:43,588 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:43,588 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:43,589 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:44,236 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:45,237 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:45,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:46,576 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:46,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:46,579 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:46,580 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:47,238 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:49,239 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:49,559 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:49,561 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:49,561 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:49,562 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:50,240 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:51,241 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:51,563 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:52,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:52,520 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:52,520 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:52,521 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:53,242 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:55,243 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:55,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:55,457 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:55,457 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:55,458 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:56,005 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:32:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:32:56,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:32:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:32:56,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:32:57,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:57,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:32:58,414 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:32:58,416 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:32:58,416 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:32:58,417 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:32:59,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:32:59,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:01,355 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:01,357 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:01,357 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:01,358 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:02,248 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:02,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:03,248 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:04,295 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:04,297 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:04,297 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:04,298 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:05,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:05,250 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:07,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:07,258 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:07,259 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:07,260 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:08,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:08,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:09,252 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:10,193 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:10,196 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:10,196 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:10,197 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:10,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:11,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:33:11,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:33:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:33:11,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:13,147 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:13,149 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:13,149 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:13,150 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:13,255 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:14,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:15,255 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:16,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:16,088 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:16,089 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:16,090 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:16,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:17,257 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:19,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:19,041 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:19,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:19,042 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:19,258 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:20,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:21,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:21,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:21,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:21,981 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:21,982 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:22,260 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:23,260 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:24,916 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:24,918 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:24,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:24,920 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:25,262 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:25,921 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:26,006 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:33:26,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:33:26,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:33:26,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:33:27,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:27,873 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:27,874 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:27,874 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:27,875 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:28,264 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:29,264 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:30,863 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:30,866 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:30,866 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:30,867 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:31,266 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:31,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:33,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:33,820 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:33,822 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:33,823 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:33,824 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:34,268 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:35,268 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:36,751 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:36,754 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:36,754 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:36,755 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:37,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:37,756 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:39,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:39,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:39,692 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:39,692 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:39,693 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:40,272 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:41,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:33:41,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:33:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:33:41,272 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:42,619 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:42,620 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:42,621 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:42,622 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:43,274 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:43,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:45,275 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:45,561 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:45,563 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:45,564 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:45,565 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:46,276 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:47,276 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:48,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:48,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:48,503 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:48,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:48,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:48,815 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:49,278 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:49,278 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:51,279 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:52,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:52,049 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:52,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:52,050 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:52,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:53,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:54,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:33:54,984 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:54,986 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:54,987 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:54,988 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:55,282 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:56,009 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:33:56,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:33:56,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:33:56,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:33:57,283 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:57,930 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:33:57,931 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:33:57,932 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:33:57,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:33:58,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:33:59,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:33:59,973 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:00,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:00,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:00,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:00,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:01,286 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:03,287 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:03,828 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:03,830 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:03,831 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:03,832 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:04,288 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:05,288 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:05,832 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:06,761 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:06,763 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:06,764 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:06,765 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:07,289 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:09,290 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:09,711 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:09,713 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:09,714 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:09,715 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:10,291 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:11,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:34:11,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:34:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:34:11,234 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:11,292 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:12,790 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:12,792 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:12,792 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:12,793 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:13,293 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:15,294 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:15,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:15,727 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:15,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:15,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:16,295 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:16,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:17,296 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:18,657 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:18,660 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:18,660 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:18,661 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:19,297 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:21,298 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:21,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:21,623 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:21,623 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:21,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:22,299 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:22,625 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:23,299 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:24,562 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:24,565 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:24,565 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:24,566 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:25,301 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:25,301 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:26,009 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:34:26,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:34:26,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:34:26,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:34:27,508 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:27,509 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:27,509 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:27,510 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:28,303 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:28,549 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:29,303 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:30,447 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:30,449 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:30,450 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:30,451 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:31,305 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:31,305 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:33,388 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:33,390 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:33,390 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:33,391 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:34,307 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:34,392 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:35,307 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:36,330 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:36,332 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:36,333 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:36,334 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:37,309 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:37,309 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:39,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:39,276 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:39,276 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:39,277 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:39,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:40,278 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:41,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:34:41,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:34:41,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:34:41,311 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:42,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:42,231 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:42,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:42,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:42,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:43,313 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:45,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:45,171 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:45,171 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:45,172 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:45,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:46,173 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:47,315 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:48,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:48,115 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:48,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:48,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:48,316 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:49,317 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:51,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:51,057 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:51,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:51,058 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:51,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:52,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:53,319 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:54,032 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:54,036 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:54,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:54,038 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:54,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:55,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:56,010 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:34:56,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:34:56,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:34:56,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:34:57,027 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:57,028 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:57,029 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:57,030 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:34:57,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:34:57,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:34:59,323 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:34:59,968 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:34:59,970 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:34:59,971 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:34:59,972 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:00,324 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:01,324 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:02,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:02,917 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:02,918 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:02,918 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:02,919 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:03,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:05,327 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:05,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:05,855 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:05,855 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:05,856 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:06,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:07,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:08,791 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:08,794 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:08,794 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:08,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:08,795 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:09,330 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:11,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:35:11,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:35:11,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:35:11,331 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:11,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:11,731 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:11,731 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:11,732 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:12,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:13,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:14,670 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:14,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:14,673 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:14,673 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:14,674 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:15,334 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:15,334 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:17,628 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:17,630 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:17,631 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:17,632 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:18,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:19,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:20,587 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:20,589 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:20,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:20,590 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:20,591 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:21,338 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:21,338 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:23,527 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:23,530 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:23,530 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:23,531 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:24,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:25,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:26,011 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:35:26,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:26,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:35:26,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:35:26,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:35:26,463 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:26,464 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:26,465 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:26,466 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:27,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:27,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:29,408 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:29,416 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:29,416 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:29,417 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:30,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:31,345 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:31,418 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:32,396 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:32,399 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:32,399 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:32,400 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:33,346 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:33,347 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:35,337 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:35,339 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:35,340 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:35,341 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:35,348 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:37,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:37,349 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:38,280 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:38,283 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:38,283 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:38,284 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:38,351 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:39,351 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:35:41,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:35:41,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:35:41,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:41,278 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:41,278 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:41,279 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:41,353 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:43,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:43,354 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:44,215 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:44,218 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:44,218 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:44,219 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:44,355 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:45,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:47,158 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:47,160 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:47,161 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:47,162 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:47,358 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:49,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:49,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:50,099 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:50,101 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:50,102 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:50,103 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:50,360 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:51,361 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:53,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:53,044 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:53,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:53,059 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:53,362 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:55,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:35:55,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:55,975 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:55,977 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:55,977 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:55,979 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:56,012 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:35:56,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:35:56,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:35:56,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:35:56,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:35:57,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:35:58,917 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:35:58,919 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:35:58,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:35:58,920 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:35:59,367 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:00,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:01,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:01,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:01,857 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:01,858 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:01,859 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:02,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:03,370 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:04,795 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:04,797 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:04,797 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:04,798 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:05,372 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:06,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:07,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:07,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:07,731 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:07,732 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:07,733 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:08,374 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:09,375 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:10,666 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:10,669 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:10,669 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:10,670 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:36:11,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:36:11,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:36:11,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:11,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:12,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:13,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:13,608 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:13,608 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:13,609 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:14,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:15,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:16,545 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:16,547 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:16,547 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:16,548 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:17,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:17,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:17,549 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:19,477 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:19,480 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:19,480 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:19,481 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:20,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:21,384 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:22,411 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:22,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:22,419 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:22,420 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:23,386 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:23,386 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:23,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:25,403 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:25,406 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:25,406 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:25,407 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:26,013 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:36:26,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:36:26,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:36:26,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:36:26,388 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:27,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:28,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:28,344 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:28,344 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:28,345 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:28,390 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:29,385 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:29,390 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:31,309 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:31,311 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:31,311 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:31,313 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:31,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:33,393 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:34,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:34,268 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:34,268 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:34,270 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:34,394 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:35,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:35,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:37,200 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:37,203 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:37,203 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:37,204 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:37,397 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:39,398 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:40,153 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:40,155 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:40,155 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:40,157 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:40,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:36:41,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:36:41,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:36:41,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:41,400 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:43,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:43,092 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:43,092 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:43,093 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:43,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:45,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:46,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:46,021 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:46,022 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:46,023 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:46,404 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:47,023 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:47,405 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:48,948 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:48,951 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:48,951 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:48,952 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:49,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:51,407 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:51,876 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:51,878 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:51,879 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:51,880 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:52,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:52,880 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:53,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:54,826 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:54,828 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:54,828 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:54,829 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:55,411 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:56,013 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:36:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:36:56,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:36:56,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:36:57,412 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:36:57,779 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:36:57,780 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:36:57,781 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:36:57,782 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:36:58,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:36:58,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:36:59,414 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:00,735 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:00,737 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:00,738 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:00,739 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:01,416 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:01,416 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:03,675 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:03,677 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:03,678 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:03,679 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:04,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:04,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:05,419 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:06,631 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:06,634 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:06,634 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:06,635 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:07,421 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:07,421 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:09,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:09,696 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:09,696 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:09,696 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:09,697 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:10,423 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:37:11,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:37:11,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:37:11,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:12,647 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:12,649 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:12,649 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:12,650 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:13,425 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:13,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:15,593 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:15,596 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:15,596 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:15,597 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:15,598 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:16,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:17,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:18,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:18,545 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:18,545 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:18,546 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:19,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:19,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:21,497 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:21,499 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:21,499 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:21,499 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:21,500 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:22,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:23,433 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:24,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:24,443 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:24,444 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:24,445 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:25,435 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:25,435 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:26,014 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:37:26,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:37:26,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:37:26,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:37:27,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:27,383 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:27,385 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:27,385 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:27,386 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:27,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:29,437 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:30,322 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:30,324 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:30,324 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:30,326 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:30,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:31,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:32,326 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:33,265 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:33,268 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:33,268 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:33,269 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:33,441 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:35,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:36,203 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:36,205 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:36,205 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:36,207 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:36,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:37,444 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:38,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:39,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:39,144 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:39,144 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:39,145 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:39,445 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:37:41,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:37:41,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:37:41,446 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:42,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:42,079 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:42,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:42,080 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:42,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:43,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:44,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:45,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:45,018 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:45,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:45,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:45,450 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:47,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:47,952 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:47,954 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:47,955 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:47,956 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:48,452 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:49,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:49,956 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:50,913 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:50,916 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:50,916 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:50,917 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:51,455 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:53,456 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:53,931 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:53,956 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:53,957 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:53,958 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:54,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:54,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:37:55,458 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:56,015 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:37:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:37:56,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:37:56,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:37:56,869 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:56,871 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:56,871 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:56,872 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:37:57,460 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:37:59,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:37:59,820 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:37:59,822 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:37:59,822 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:37:59,823 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:00,462 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:00,828 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:01,463 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:02,769 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:02,771 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:02,771 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:02,772 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:03,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:03,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:05,705 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:05,707 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:05,708 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:05,709 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:06,467 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:06,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:07,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:08,644 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:08,646 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:08,646 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:08,647 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:09,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:09,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:11,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:38:11,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:38:11,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:38:11,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:11,586 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:11,587 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:11,588 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:12,471 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:12,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:13,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:14,533 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:14,535 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:14,535 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:14,536 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:15,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:15,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:17,470 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:17,472 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:17,473 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:17,474 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:17,475 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:18,474 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:19,477 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:20,410 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:20,412 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:20,413 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:20,414 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:20,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:21,479 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:23,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:23,350 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:23,351 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:23,352 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:23,480 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:24,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:25,481 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:26,016 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:38:26,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:38:26,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:38:26,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:38:26,291 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:26,293 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:26,293 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:26,294 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:26,483 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:27,483 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:29,246 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:29,250 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:29,250 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:29,252 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:29,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:30,252 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:31,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:32,194 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:32,197 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:32,197 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:32,198 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:32,487 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:33,488 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:35,230 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:35,232 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:35,232 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:35,234 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:35,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:36,234 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:37,491 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:38,184 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:38,186 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:38,186 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:38,187 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:38,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:39,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:41,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:38:41,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:38:41,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:38:41,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:41,266 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:41,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:41,266 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:41,268 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:41,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:43,495 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:44,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:44,062 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:44,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:44,064 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:44,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:45,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:47,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:47,003 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:47,003 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:47,003 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:47,004 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:47,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:49,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:49,941 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:49,944 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:49,944 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:49,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:50,499 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:51,500 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:52,906 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:52,908 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:52,909 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:52,909 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:52,910 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:53,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:53,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:55,849 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:55,851 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:55,851 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:55,853 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:56,017 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:38:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:38:56,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:38:56,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:38:56,503 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:38:57,504 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:58,247 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:38:58,809 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:38:58,811 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:38:58,811 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:38:58,812 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:38:59,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:38:59,506 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:01,770 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:01,773 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:01,773 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:01,774 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:02,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:03,508 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:03,775 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:04,731 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:04,733 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:04,734 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:04,735 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:05,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:05,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:07,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:07,681 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:07,682 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:07,683 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:08,511 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:09,512 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:09,683 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:10,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:10,636 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:10,636 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:10,638 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:11,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:39:11,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:39:11,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:39:11,513 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:11,513 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:13,570 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:13,571 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:13,572 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:13,573 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:14,515 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:15,516 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:15,613 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:16,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:16,507 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:16,508 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:16,509 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:16,517 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:17,518 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:19,534 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:19,536 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:19,536 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:19,537 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:20,520 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:21,520 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:21,538 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:22,477 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:22,479 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:22,479 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:22,481 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:22,522 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:23,522 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:25,414 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:25,416 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:25,416 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:25,417 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:25,524 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:26,019 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:39:26,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:39:26,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:39:26,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:39:27,305 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:27,525 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:28,383 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:28,385 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:28,385 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:28,386 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:28,526 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:29,527 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:31,325 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:31,327 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:31,328 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:31,329 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:31,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:32,330 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:33,529 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:34,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:34,295 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:34,295 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:34,296 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:34,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:35,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:37,231 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:37,233 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:37,234 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:37,235 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:37,533 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:38,235 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:39,534 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:40,169 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:40,171 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:40,171 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:40,172 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:40,535 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:41,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:39:41,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:39:41,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:39:41,536 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:43,102 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:43,104 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:43,104 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:43,105 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:43,538 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:44,166 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:45,538 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:46,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:46,040 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:46,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:46,041 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:46,539 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:47,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:48,988 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:48,991 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:48,991 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:48,992 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:49,541 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:49,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:49,993 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:51,925 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:51,928 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:51,928 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:51,929 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:52,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:53,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:54,860 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:54,863 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:54,863 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:54,864 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:55,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:39:55,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:55,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:39:56,018 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:39:56,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:39:56,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:39:56,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:39:57,796 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:39:57,798 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:39:57,798 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:39:57,800 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:39:58,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:39:59,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:00,825 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:00,828 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:00,828 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:00,829 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:01,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:01,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:01,830 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:03,787 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:03,789 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:03,790 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:03,791 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:04,552 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:05,553 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:06,726 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:06,728 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:06,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:06,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:07,554 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:07,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:07,730 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:09,662 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:09,666 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:09,667 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:09,668 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:10,556 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:11,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:40:11,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:40:11,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:40:11,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:12,616 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:12,617 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:12,618 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:12,619 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:13,558 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:13,558 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:13,657 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:15,550 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:15,552 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:15,553 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:15,554 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:15,560 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:17,561 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:18,499 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:18,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:18,503 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:18,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:18,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:19,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:19,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:21,445 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:21,447 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:21,447 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:21,449 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:21,563 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:23,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:24,390 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:24,392 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:24,392 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:24,393 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:24,565 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:25,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:25,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:26,019 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:40:26,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:40:26,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:40:26,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:40:27,337 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:27,339 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:27,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:27,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:27,568 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:29,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:30,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:30,275 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:30,276 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:30,277 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:30,570 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:31,278 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:32,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:33,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:33,227 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:33,227 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:33,228 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:33,572 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:36,159 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:36,161 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:36,162 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:36,163 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:36,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:36,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:37,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:38,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:39,104 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:39,106 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:39,107 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:39,108 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:39,576 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:41,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:40:41,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:40:41,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:40:42,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:42,051 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:42,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:42,052 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:42,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:42,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:43,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:44,579 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:45,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:45,023 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:45,023 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:45,024 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:45,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:46,581 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:48,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:48,058 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:48,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:48,060 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:48,583 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:49,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:50,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:51,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:51,021 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:51,022 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:51,023 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:51,585 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:52,585 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:53,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:53,976 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:53,976 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:53,977 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:54,587 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:54,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:40:56,020 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:40:56,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:40:56,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:40:56,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:40:56,588 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:56,921 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:56,922 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:56,923 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:56,924 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:40:57,589 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:40:58,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:40:59,864 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:40:59,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:40:59,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:40:59,869 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:00,591 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:00,870 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:02,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:02,813 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:02,815 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:02,815 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:02,816 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:03,593 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:04,594 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:05,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:05,786 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:05,786 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:05,787 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:06,595 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:06,788 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:08,597 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:08,749 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:08,752 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:08,753 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:08,754 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:09,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:10,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:11,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:41:11,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:41:11,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:41:11,701 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:11,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:11,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:11,704 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:12,600 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:12,745 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:14,601 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:14,670 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:14,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:14,673 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:14,674 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:15,602 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:16,603 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:17,626 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:17,628 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:17,628 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:17,629 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:18,604 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:18,630 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:20,577 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:20,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:20,579 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:20,580 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:20,606 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:20,606 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:22,607 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:23,521 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:23,523 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:23,524 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:23,525 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:23,608 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:24,525 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:26,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:41:26,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:41:26,042 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:41:26,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:41:26,462 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:26,464 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:26,464 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:26,465 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:26,610 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:26,610 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:28,611 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:29,508 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:29,511 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:29,511 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:29,512 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:29,612 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:30,513 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:32,447 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:32,450 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:32,450 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:32,451 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:32,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:32,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:34,616 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:35,382 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:35,385 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:35,385 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:35,386 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:35,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:36,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:38,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:38,319 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:38,319 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:38,320 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:38,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:38,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:40,620 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:41,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:41:41,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:41:41,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:41:41,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:41,277 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:41,278 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:41,279 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:41,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:42,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:44,228 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:44,230 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:44,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:44,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:44,623 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:44,623 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:46,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:47,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:47,180 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:47,180 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:47,181 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:47,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:48,181 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:50,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:50,137 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:50,137 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:50,139 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:50,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:50,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:52,628 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:53,087 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:53,091 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:53,091 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:53,092 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:53,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:54,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:41:54,630 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:56,021 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:41:56,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:41:56,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:41:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:41:56,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:56,243 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:56,243 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:56,244 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:56,632 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:41:58,633 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:41:59,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:41:59,002 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:41:59,002 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:41:59,003 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:41:59,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:00,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:00,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:01,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:01,948 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:01,949 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:01,950 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:02,637 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:04,638 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:04,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:04,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:04,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:04,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:05,639 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:05,893 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:06,640 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:07,830 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:07,833 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:07,833 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:07,834 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:08,642 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:10,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:10,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:10,779 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:10,779 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:10,780 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:11,040 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:42:11,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:42:11,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:42:11,222 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:11,644 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:12,645 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:13,816 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:13,818 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:13,818 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:13,819 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:14,647 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:16,648 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:16,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:16,806 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:16,807 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:16,807 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:16,808 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:17,649 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:18,650 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:19,741 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:19,743 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:19,743 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:19,744 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:20,652 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:22,653 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:22,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:22,684 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:22,685 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:22,685 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:22,686 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:23,654 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:24,655 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:25,643 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:25,646 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:25,646 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:25,647 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:25,656 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:26,022 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:42:26,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:42:26,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:42:26,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:42:28,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:28,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:28,581 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:28,581 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:28,582 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:28,658 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:28,658 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:30,659 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:31,512 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:31,515 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:31,515 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:31,516 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:31,660 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:33,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:34,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:34,456 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:34,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:34,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:34,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:34,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:36,664 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:37,391 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:37,394 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:37,394 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:37,395 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:37,665 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:39,396 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:40,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:40,335 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:40,336 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:40,338 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:40,667 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:40,667 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:41,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:42:41,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:42:41,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:42:42,668 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:43,277 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:43,279 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:43,279 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:43,280 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:43,669 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:45,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:46,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:46,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:46,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:46,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:46,671 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:46,672 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:48,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:49,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:49,145 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:49,146 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:49,147 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:49,674 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:50,675 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:51,148 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:52,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:52,076 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:52,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:52,077 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:52,676 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:54,678 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:55,108 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:55,111 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:55,111 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:55,112 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:55,679 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:42:56,023 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:42:56,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:42:56,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:42:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:42:56,310 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:42:56,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:42:58,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:42:58,061 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:42:58,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:42:58,063 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:42:58,681 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:00,683 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:01,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:01,043 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:01,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:01,044 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:01,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:02,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:02,685 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:03,993 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:03,996 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:03,996 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:03,998 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:04,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:06,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:06,995 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:07,002 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:07,003 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:07,004 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:07,689 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:08,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:08,690 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:09,983 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:09,985 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:09,986 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:09,987 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:10,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:11,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:43:11,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:43:11,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:43:12,693 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:12,963 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:12,964 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:12,964 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:12,965 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:13,005 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:13,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:14,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:15,909 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:15,911 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:15,912 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:15,913 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:16,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:18,697 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:18,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:18,854 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:18,855 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:18,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:18,856 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:19,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:20,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:21,812 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:21,814 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:21,814 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:21,815 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:22,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:24,701 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:24,753 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:24,755 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:24,755 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:24,756 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:24,757 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:25,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:26,025 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:43:26,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:43:26,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:43:26,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:43:26,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:27,695 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:27,697 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:27,697 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:27,699 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:27,703 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:30,637 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:30,640 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:30,641 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:30,641 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:30,642 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:30,705 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:30,705 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:32,706 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:33,627 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:33,629 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:33,630 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:33,631 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:33,707 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:36,588 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:36,591 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:36,591 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:36,591 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:36,592 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:36,709 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:36,709 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:38,710 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:39,636 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:39,638 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:39,639 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:39,640 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:39,711 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:39,890 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:40,712 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:41,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:43:41,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:43:41,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:43:42,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:42,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:43,015 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:43,017 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:43,017 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:43,018 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:43,714 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:44,714 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:45,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:45,963 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:45,963 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:45,964 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:46,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:47,965 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:48,717 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:48,900 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:48,902 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:48,902 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:48,903 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:49,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:50,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:51,839 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:51,841 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:51,841 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:51,842 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:52,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:53,843 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:43:54,721 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:54,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:54,802 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:54,802 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:54,803 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:55,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:56,025 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:43:56,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:43:56,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:43:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:43:56,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:43:57,731 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:43:57,733 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:43:57,734 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:43:57,735 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:43:58,724 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:43:59,773 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:00,678 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:00,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:00,704 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:00,705 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:00,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:00,726 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:02,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:03,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:03,620 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:03,621 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:03,622 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:03,728 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:05,622 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:06,563 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:06,566 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:06,566 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:06,567 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:06,730 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:06,730 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:08,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:09,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:09,549 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:09,549 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:09,551 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:09,732 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:11,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:44:11,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:44:11,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:44:11,265 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:12,495 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:12,497 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:12,497 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:12,498 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:12,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:12,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:14,735 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:15,447 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:15,450 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:15,451 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:15,453 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:15,736 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:16,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:18,389 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:18,391 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:18,391 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:18,392 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:18,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:18,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:20,739 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:21,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:21,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:21,323 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:21,324 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:21,740 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:22,325 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:22,740 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:24,384 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:24,386 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:24,387 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:24,388 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:24,742 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:26,026 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:44:26,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:44:26,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:44:26,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:44:26,743 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:27,360 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:27,361 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:27,362 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:27,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:27,363 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:27,744 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:30,315 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:30,317 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:30,318 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:30,319 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:30,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:30,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:32,748 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:33,305 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:33,307 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:33,308 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:33,308 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:33,309 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:33,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:34,750 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:36,287 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:36,289 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:36,289 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:36,291 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:36,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:38,752 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:39,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:39,262 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:39,262 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:39,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:39,264 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:39,754 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:40,754 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:41,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:44:41,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:44:41,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:44:42,226 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:42,227 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:42,228 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:42,229 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:42,756 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:44,269 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:44,757 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:45,178 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:45,186 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:45,186 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:45,187 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:45,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:46,760 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:48,111 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:48,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:48,114 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:48,115 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:48,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:50,116 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:50,763 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:51,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:51,046 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:51,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:51,047 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:51,764 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:52,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:53,982 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:53,985 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:53,985 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:53,987 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:54,766 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:55,987 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:44:56,027 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:44:56,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:44:56,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:44:56,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:44:56,767 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:56,931 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:56,933 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:56,933 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:56,934 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:44:57,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:44:58,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:44:59,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:44:59,894 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:44:59,894 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:44:59,895 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:00,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:01,896 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:02,772 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:02,864 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:02,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:02,868 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:02,869 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:03,774 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:04,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:05,823 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:05,825 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:05,826 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:05,827 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:06,776 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:07,827 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:08,778 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:08,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:08,803 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:08,804 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:08,805 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:09,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:10,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:11,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:45:11,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:45:11,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:45:11,770 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:11,772 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:11,772 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:11,773 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:11,781 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:13,813 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:14,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:14,701 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:14,702 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:14,703 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:14,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:14,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:16,784 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:17,641 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:17,643 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:17,644 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:17,645 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:17,785 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:19,645 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:20,573 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:20,575 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:20,575 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:20,576 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:20,787 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:20,787 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:22,788 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:23,520 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:23,523 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:23,523 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:23,524 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:23,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:25,525 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:26,028 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:45:26,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:45:26,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:45:26,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:45:26,461 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:26,463 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:26,463 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:26,464 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:26,791 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:26,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:28,793 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:29,437 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:29,440 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:29,440 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:29,441 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:29,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:30,795 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:31,442 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:32,377 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:32,379 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:32,379 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:32,380 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:32,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:34,797 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:35,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:35,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:35,338 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:35,339 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:35,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:36,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:37,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:38,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:38,297 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:38,297 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:38,299 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:38,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:40,802 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:41,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:45:41,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:45:41,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:45:41,234 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:41,313 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:41,314 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:41,315 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:41,803 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:42,804 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:43,315 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:44,217 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:44,219 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:44,219 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:44,220 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:44,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:46,807 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:47,172 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:47,175 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:47,175 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:47,176 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:47,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:48,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:49,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:50,125 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:50,129 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:50,129 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:50,130 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:50,810 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:52,811 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:53,170 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:53,175 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:53,175 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:53,176 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:53,812 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:54,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:55,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:45:56,028 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:45:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:45:56,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:45:56,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:45:56,119 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:56,243 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:56,243 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:56,244 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:56,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:45:58,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:45:59,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:45:59,074 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:45:59,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:45:59,076 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:45:59,817 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:00,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:01,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:02,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:02,018 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:02,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:02,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:02,819 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:04,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:04,992 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:04,994 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:04,995 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:04,996 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:05,821 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:06,822 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:06,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:07,939 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:07,942 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:07,942 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:07,943 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:08,823 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:10,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:10,874 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:10,876 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:10,876 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:10,878 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:11,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:46:11,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:46:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:46:11,825 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:12,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:12,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:13,813 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:13,815 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:13,815 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:13,816 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:13,827 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:16,757 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:16,759 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:16,759 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:16,761 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:16,829 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:16,829 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:17,761 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:18,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:19,732 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:19,734 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:19,735 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:19,736 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:19,831 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:22,688 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:22,690 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:22,691 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:22,692 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:22,833 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:22,833 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:23,692 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:24,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:25,630 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:25,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:25,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:25,634 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:25,835 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:26,034 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:46:26,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:46:26,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:46:26,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:46:28,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:28,583 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:28,584 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:28,585 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:28,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:28,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:29,625 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:30,839 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:31,515 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:31,517 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:31,518 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:31,519 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:31,840 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:32,840 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:34,451 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:34,453 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:34,453 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:34,455 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:34,842 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:35,456 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:36,843 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:37,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:37,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:37,483 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:37,484 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:37,844 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:38,844 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:40,442 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:40,444 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:40,445 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:40,447 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:40,846 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:41,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:46:41,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:46:41,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:46:41,216 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:42,847 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:43,393 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:43,395 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:43,395 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:43,396 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:43,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:44,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:46,359 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:46,361 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:46,361 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:46,361 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:46,363 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:46,850 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:48,851 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:49,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:49,300 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:49,300 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:49,302 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:49,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:50,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:52,238 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:52,240 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:52,240 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:52,240 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:52,242 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:52,854 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:54,855 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:55,187 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:55,189 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:55,190 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:55,191 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:55,856 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:46:56,030 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:46:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:46:56,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:46:56,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:46:56,856 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:46:58,130 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:46:58,131 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:46:58,132 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:46:58,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:46:58,133 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:46:58,858 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:00,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:01,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:01,079 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:01,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:01,080 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:01,860 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:02,860 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:04,017 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:04,019 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:04,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:04,020 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:04,021 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:04,862 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:06,863 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:06,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:06,957 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:06,957 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:06,958 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:07,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:08,865 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:09,893 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:09,895 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:09,896 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:09,896 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:09,897 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:10,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:11,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:47:11,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:47:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:47:12,832 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:12,833 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:12,834 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:12,835 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:12,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:12,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:14,869 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:15,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:15,823 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:15,824 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:15,824 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:15,825 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:15,870 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:18,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:18,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:18,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:18,868 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:18,872 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:18,872 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:20,869 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:20,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:21,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:21,788 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:21,788 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:21,790 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:21,874 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:24,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:24,731 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:24,731 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:24,732 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:24,876 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:24,876 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:26,031 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:47:26,031 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:26,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:47:26,042 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:47:26,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:47:26,877 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:27,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:27,665 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:27,666 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:27,667 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:27,878 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:30,601 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:30,603 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:30,603 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:30,604 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:30,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:30,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:31,605 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:32,881 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:33,558 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:33,560 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:33,561 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:33,562 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:33,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:34,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:36,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:36,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:36,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:36,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:36,884 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:37,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:38,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:39,449 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:39,452 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:39,452 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:39,453 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:39,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:40,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:41,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:47:41,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:47:41,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:47:42,410 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:42,411 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:42,412 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:42,413 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:42,888 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:43,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:44,888 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:45,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:45,354 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:45,354 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:45,355 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:45,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:46,890 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:48,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:48,295 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:48,296 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:48,297 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:48,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:49,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:50,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:51,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:51,241 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:51,241 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:51,243 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:51,893 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:52,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:54,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:54,214 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:54,214 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:54,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:54,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:55,216 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:47:56,031 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:47:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:47:56,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:47:56,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:47:56,897 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:47:57,190 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:47:57,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:47:57,192 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:47:57,193 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:47:57,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:47:58,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:00,134 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:00,136 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:00,137 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:00,138 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:00,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:01,139 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:02,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:03,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:03,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:03,170 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:03,171 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:03,902 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:04,902 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:06,150 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:06,153 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:06,153 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:06,153 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:06,154 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:06,903 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:08,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:09,097 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:09,108 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:09,108 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:09,109 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:09,905 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:10,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:11,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:48:11,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:48:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:48:11,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:12,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:12,039 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:12,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:12,040 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:12,907 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:14,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:14,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:14,979 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:14,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:14,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:15,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:16,910 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:16,981 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:17,917 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:17,919 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:17,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:17,920 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:18,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:20,858 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:20,864 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:20,864 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:20,865 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:20,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:20,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:22,866 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:22,914 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:23,798 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:23,800 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:23,800 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:23,801 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:23,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:26,032 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:48:26,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:48:26,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:48:26,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:48:26,744 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:26,746 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:26,746 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:26,747 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:26,916 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:26,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:28,785 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:28,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:29,704 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:29,706 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:29,706 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:29,707 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:29,918 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:32,667 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:32,669 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:32,670 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:32,671 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:32,920 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:32,921 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:34,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:34,922 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:35,828 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:35,830 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:35,831 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:35,832 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:35,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:38,759 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:38,761 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:38,761 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:38,762 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:38,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:38,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:39,763 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:40,926 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:41,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:48:41,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:48:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:48:41,712 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:41,714 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:41,714 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:41,715 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:41,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:44,675 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:44,677 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:44,677 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:44,678 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:44,929 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:44,929 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:45,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:46,930 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:47,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:47,691 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:47,691 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:47,692 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:47,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:48,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:50,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:50,610 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:50,610 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:50,611 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:50,932 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:51,612 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:52,933 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:53,537 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:53,539 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:53,540 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:53,541 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:53,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:54,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:56,033 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:48:56,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:48:56,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:48:56,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:48:56,475 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:56,476 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:56,477 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:56,478 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:56,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:48:57,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:48:58,937 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:48:59,401 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:48:59,408 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:48:59,409 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:48:59,410 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:48:59,938 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:00,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:02,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:02,344 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:02,344 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:02,346 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:02,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:03,346 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:04,941 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:05,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:05,276 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:05,277 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:05,278 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:05,942 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:06,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:08,203 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:08,206 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:08,206 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:08,207 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:08,944 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:09,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:10,945 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:11,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:49:11,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:49:11,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:49:11,146 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:11,257 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:11,258 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:11,259 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:11,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:12,947 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:14,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:14,078 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:14,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:14,080 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:14,948 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:15,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:16,949 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:17,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:17,006 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:17,007 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:17,008 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:17,950 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:18,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:19,942 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:19,944 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:19,944 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:19,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:19,952 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:20,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:22,874 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:22,876 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:22,876 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:22,877 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:22,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:22,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:24,955 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:25,824 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:25,827 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:25,828 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:25,829 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:25,956 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:26,034 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:49:26,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:26,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:49:26,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:49:26,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:49:28,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:28,854 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:28,854 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:28,855 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:28,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:28,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:30,959 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:31,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:31,780 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:31,780 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:31,780 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:31,781 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:31,959 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:34,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:34,711 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:34,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:34,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:34,961 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:34,961 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:36,962 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:37,650 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:37,652 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:37,652 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:37,653 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:37,654 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:37,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:38,964 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:40,580 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:40,582 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:40,583 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:40,584 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:40,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:41,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:49:41,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:49:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:49:42,966 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:43,250 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:43,526 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:43,528 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:43,528 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:43,529 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:43,967 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:44,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:46,462 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:46,464 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:46,464 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:46,466 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:46,969 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:48,467 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:48,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:49,395 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:49,398 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:49,398 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:49,399 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:49,971 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:50,971 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:52,328 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:52,330 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:52,331 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:52,332 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:52,973 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:54,332 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:49:54,974 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:55,258 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:55,260 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:55,261 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:55,262 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:55,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:49:56,035 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:49:56,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:49:56,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:49:56,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:49:56,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:49:58,200 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:49:58,202 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:49:58,202 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:49:58,203 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:49:58,977 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:00,242 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:00,978 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:01,129 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:01,154 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:01,155 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:01,156 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:01,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:02,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:04,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:04,096 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:04,096 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:04,097 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:04,981 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:06,098 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:06,982 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:07,029 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:07,031 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:07,032 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:07,034 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:07,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:08,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:09,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:09,998 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:09,998 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:09,999 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:10,985 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:50:11,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:50:11,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:50:11,242 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:12,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:13,021 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:13,022 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:13,022 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:13,024 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:13,987 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:14,987 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:15,941 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:15,943 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:15,943 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:15,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:15,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:16,945 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:18,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:18,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:18,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:18,868 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:18,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:18,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:20,991 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:21,788 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:21,790 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:21,791 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:21,792 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:21,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:22,793 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:22,993 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:24,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:24,711 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:24,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:24,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:24,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:26,035 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:50:26,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:50:26,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:50:26,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:50:26,995 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:27,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:27,630 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:27,631 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:27,632 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:27,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:28,673 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:28,997 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:30,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:30,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:30,557 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:30,558 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:30,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:32,999 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:33,480 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:33,482 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:33,482 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:33,483 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:34,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:34,484 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:35,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:36,421 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:36,423 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:36,424 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:36,425 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:37,002 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:39,003 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:39,361 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:39,364 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:39,364 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:39,365 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:40,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:40,403 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:41,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:41,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:50:41,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:50:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:50:42,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:42,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:42,292 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:42,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:43,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:45,007 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:45,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:45,214 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:45,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:45,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:46,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:46,216 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:47,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:48,149 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:48,151 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:48,151 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:48,152 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:49,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:51,011 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:51,088 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:51,090 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:51,091 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:51,092 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:52,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:52,092 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:53,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:54,103 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:54,106 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:54,106 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:54,107 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:55,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:56,036 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:50:56,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:50:56,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:50:56,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:50:57,015 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:57,030 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:57,031 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:57,031 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:57,032 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:50:58,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:50:58,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:50:59,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:50:59,967 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:50:59,970 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:50:59,970 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:50:59,971 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:00,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:02,900 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:02,904 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:02,905 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:02,906 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:03,019 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:03,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:03,906 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:05,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:05,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:05,843 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:05,843 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:05,845 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:06,021 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:07,022 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:08,778 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:08,780 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:08,781 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:08,782 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:09,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:09,783 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:11,024 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:51:11,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:51:11,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:51:11,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:11,726 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:11,726 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:11,727 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:12,025 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:13,026 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:14,695 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:14,697 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:14,698 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:14,699 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:15,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:15,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:17,028 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:17,615 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:17,617 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:17,618 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:17,619 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:18,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:19,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:20,541 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:20,543 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:20,543 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:20,545 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:21,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:21,546 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:23,032 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:23,487 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:23,489 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:23,490 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:23,491 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:24,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:25,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:26,037 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:51:26,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:51:26,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:51:26,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:51:26,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:26,422 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:26,422 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:26,423 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:27,035 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:27,461 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:29,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:29,366 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:29,374 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:29,374 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:29,375 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:30,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:31,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:32,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:32,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:32,324 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:32,325 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:33,039 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:33,325 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:35,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:35,252 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:35,254 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:35,254 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:35,255 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:36,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:37,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:38,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:38,274 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:38,274 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:38,276 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:39,043 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:39,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:41,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:51:41,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:51:41,046 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:41,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:51:41,202 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:41,251 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:41,252 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:41,253 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:42,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:43,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:44,122 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:44,123 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:44,124 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:44,125 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:45,049 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:45,165 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:47,050 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:47,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:47,063 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:47,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:47,065 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:48,050 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:49,051 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:49,979 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:49,981 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:49,982 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:49,983 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:50,052 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:50,983 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:52,955 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:52,958 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:52,958 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:52,959 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:53,054 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:53,054 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:55,055 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:55,892 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:55,894 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:55,894 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:55,895 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:56,038 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:51:56,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:51:56,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:51:56,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:51:56,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:51:56,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:51:57,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:51:58,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:51:58,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:51:58,837 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:51:58,838 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:51:59,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:01,059 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:01,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:01,786 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:01,786 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:01,787 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:01,788 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:02,060 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:03,060 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:04,716 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:04,719 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:04,719 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:04,720 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:05,062 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:07,063 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:07,666 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:07,668 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:07,669 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:07,669 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:07,670 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:08,064 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:09,064 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:10,610 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:10,612 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:10,613 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:10,614 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:52:11,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:52:11,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:52:11,066 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:13,067 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:13,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:13,540 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:13,541 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:13,542 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:13,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:14,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:15,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:16,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:16,484 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:16,484 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:16,485 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:17,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:18,486 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:19,071 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:19,435 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:19,440 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:19,440 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:19,444 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:20,072 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:21,072 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:22,456 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:22,465 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:22,465 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:22,467 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:23,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:24,468 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:25,075 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:25,396 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:25,398 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:25,398 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:25,400 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:26,039 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:52:26,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:52:26,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:52:26,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:52:26,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:27,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:28,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:28,322 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:28,322 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:28,323 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:29,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:30,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:31,079 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:31,278 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:31,280 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:31,281 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:31,282 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:32,080 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:33,080 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:34,217 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:34,220 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:34,221 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:34,222 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:35,082 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:36,222 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:37,083 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:37,149 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:37,151 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:37,152 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:37,153 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:38,084 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:39,084 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:40,096 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:40,098 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:40,099 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:40,100 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:52:41,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:52:41,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:52:41,086 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:41,264 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:43,022 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:43,024 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:43,024 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:43,025 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:43,087 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:43,087 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:45,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:45,951 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:45,953 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:45,954 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:45,955 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:46,089 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:46,955 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:47,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:48,877 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:48,879 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:48,880 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:48,881 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:49,091 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:51,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:51,807 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:51,809 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:51,809 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:51,810 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:52,093 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:52,811 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:53,093 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:54,739 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:54,743 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:54,743 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:54,749 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:55,095 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:56,040 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:52:56,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:52:56,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:52:56,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:52:57,096 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:52:57,666 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:52:57,668 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:52:57,668 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:52:57,669 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:52:58,097 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:52:58,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:52:59,097 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:00,593 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:00,595 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:00,596 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:00,597 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:01,099 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:03,100 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:03,528 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:03,531 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:03,531 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:03,532 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:04,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:04,532 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:05,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:06,549 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:06,551 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:06,552 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:06,553 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:07,103 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:09,104 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:09,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:09,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:09,484 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:09,485 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:10,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:10,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:11,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:53:11,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:53:11,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:53:11,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:12,416 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:12,423 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:12,423 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:12,425 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:13,106 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:15,107 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:15,346 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:15,348 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:15,349 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:15,350 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:16,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:16,350 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:17,109 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:18,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:18,277 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:18,278 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:18,279 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:19,110 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:21,111 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:21,232 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:21,234 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:21,235 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:21,236 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:22,112 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:22,237 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:23,113 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:24,169 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:24,172 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:24,172 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:24,173 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:25,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:26,041 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:53:26,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:53:26,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:53:26,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:53:27,095 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:27,097 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:27,097 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:27,098 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:27,116 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:27,116 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:28,137 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:29,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:30,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:30,021 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:30,021 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:30,023 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:30,118 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:30,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:31,118 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:33,120 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:33,458 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:33,460 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:33,462 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:33,462 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:33,463 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:34,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:35,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:36,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:36,396 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:36,396 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:36,398 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:37,123 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:39,124 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:39,316 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:39,319 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:39,319 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:39,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:39,320 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:40,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:53:41,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:53:41,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:53:41,126 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:42,241 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:42,243 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:42,243 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:42,244 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:43,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:45,129 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:45,164 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:45,166 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:45,167 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:45,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:45,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:46,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:47,131 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:48,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:48,223 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:48,223 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:48,224 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:49,132 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:50,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:51,133 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:51,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:51,154 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:51,154 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:51,156 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:52,135 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:53,135 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:54,096 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:54,099 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:54,099 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:54,100 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:54,137 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:56,041 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:53:56,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:53:56,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:53:56,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:53:56,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:53:57,025 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:57,032 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:57,032 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:57,033 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:53:57,139 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:57,139 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:53:59,140 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:53:59,956 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:53:59,959 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:53:59,959 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:53:59,960 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:00,141 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:01,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:01,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:02,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:02,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:02,892 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:02,893 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:03,144 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:05,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:05,822 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:05,824 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:05,825 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:05,826 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:06,146 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:07,147 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:07,826 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:08,754 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:08,756 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:08,757 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:08,758 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:09,148 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:11,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:54:11,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:54:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:54:11,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:11,692 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:11,694 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:11,694 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:11,695 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:12,151 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:13,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:13,733 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:14,633 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:14,635 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:14,636 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:14,637 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:15,153 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:17,155 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:17,602 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:17,605 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:17,606 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:17,608 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:18,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:19,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:19,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:20,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:20,538 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:20,538 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:20,540 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:21,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:23,159 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:23,480 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:23,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:23,483 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:23,484 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:24,160 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:25,161 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:25,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:26,042 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:54:26,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:54:26,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:54:26,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:54:26,412 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:26,415 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:26,415 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:26,416 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:27,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:29,164 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:29,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:29,335 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:29,335 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:29,336 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:30,165 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:31,166 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:31,336 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:32,345 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:32,347 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:32,348 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:32,349 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:33,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:35,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:35,274 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:35,277 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:35,277 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:35,278 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:36,170 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:37,171 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:37,279 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:38,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:38,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:38,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:38,217 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:39,173 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:54:41,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:54:41,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:54:41,155 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:41,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:41,297 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:41,297 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:41,298 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:42,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:42,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:43,176 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:44,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:44,083 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:44,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:44,085 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:44,177 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:45,177 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:47,028 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:47,030 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:47,030 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:47,031 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:47,179 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:48,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:49,180 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:49,976 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:49,978 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:49,978 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:49,979 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:50,181 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:51,182 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:52,904 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:52,906 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:52,906 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:52,908 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:53,184 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:53,908 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:54:55,185 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:55,844 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:55,846 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:55,847 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:55,848 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:56,043 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:54:56,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:54:56,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:54:56,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:54:56,186 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:57,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:54:58,773 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:54:58,776 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:54:58,776 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:54:58,777 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:54:59,188 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:54:59,778 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:01,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:01,706 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:01,710 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:01,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:01,713 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:02,190 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:03,191 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:04,647 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:04,649 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:04,650 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:04,651 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:05,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:05,651 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:07,194 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:07,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:07,577 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:07,577 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:07,578 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:08,195 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:09,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:10,503 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:10,505 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:10,506 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:10,507 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:11,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:55:11,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:55:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:55:11,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:11,323 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:13,198 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:13,436 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:13,438 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:13,438 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:13,439 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:14,199 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:15,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:16,440 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:16,468 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:16,470 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:16,471 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:16,472 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:17,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:19,203 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:19,417 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:19,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:19,420 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:19,421 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:20,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:21,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:22,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:22,351 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:22,351 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:22,351 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:22,352 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:23,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:25,207 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:25,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:25,272 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:25,273 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:25,273 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:26,044 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:55:26,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:55:26,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:55:26,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:55:26,208 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:27,208 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:28,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:28,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:28,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:28,215 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:28,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:29,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:31,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:31,139 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:31,139 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:31,140 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:31,211 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:31,211 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:33,212 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:34,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:34,062 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:34,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:34,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:34,064 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:34,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:35,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:36,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:36,987 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:36,988 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:36,989 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:37,215 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:39,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:39,898 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:39,901 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:39,901 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:39,901 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:39,902 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:40,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:41,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:55:41,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:55:41,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:55:41,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:42,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:42,823 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:42,824 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:42,825 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:43,219 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:45,220 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:45,741 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:45,743 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:45,744 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:45,744 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:45,745 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:46,221 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:47,222 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:48,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:48,664 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:48,664 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:48,665 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:49,223 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:51,224 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:51,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:51,643 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:51,644 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:51,644 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:51,645 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:52,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:53,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:54,514 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:54,516 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:54,516 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:54,517 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:55,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:56,044 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:55:56,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:55:56,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:55:56,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:55:57,229 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:55:57,244 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:55:57,455 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:55:57,458 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:55:57,458 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:55:57,459 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:55:58,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:55:59,231 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:00,473 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:00,476 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:00,476 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:00,477 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:01,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:02,478 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:03,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:03,412 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:03,414 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:03,415 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:03,416 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:04,247 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:05,248 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:06,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:06,343 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:06,343 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:06,344 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:07,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:08,345 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:09,250 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:09,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:09,276 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:09,276 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:09,277 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:10,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:11,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:56:11,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:56:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:56:11,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:12,204 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:12,206 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:12,206 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:12,207 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:12,252 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:13,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:14,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:15,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:15,138 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:15,139 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:15,140 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:15,254 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:17,255 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:18,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:18,072 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:18,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:18,074 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:18,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:19,257 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:20,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:21,002 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:21,005 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:21,005 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:21,006 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:21,258 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:23,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:23,943 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:23,946 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:23,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:23,947 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:24,260 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:25,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:25,947 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:26,045 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:56:26,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:56:26,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:56:26,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:56:26,885 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:26,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:26,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:26,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:27,262 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:29,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:29,806 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:29,808 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:29,809 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:29,810 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:30,265 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:31,265 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:31,810 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:32,728 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:32,730 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:32,730 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:32,731 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:33,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:35,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:35,673 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:35,675 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:35,676 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:35,677 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:36,268 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:37,269 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:37,677 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:38,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:38,608 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:38,609 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:38,610 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:39,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:41,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:56:41,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:56:41,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:56:41,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:41,624 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:41,626 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:41,626 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:41,627 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:42,272 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:43,273 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:43,669 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:44,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:44,549 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:44,550 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:44,551 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:45,274 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:47,275 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:47,472 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:47,474 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:47,475 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:47,476 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:48,276 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:49,277 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:49,476 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:50,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:50,397 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:50,397 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:50,398 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:51,278 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:53,279 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:53,330 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:53,332 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:53,332 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:53,333 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:54,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:55,281 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:55,334 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:56:56,046 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:56:56,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:56:56,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:56:56,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:56:56,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:56,299 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:56,300 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:56,301 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:57,282 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:56:57,282 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:56:59,211 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:56:59,212 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:56:59,213 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:56:59,214 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:56:59,283 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:01,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:01,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:02,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:02,143 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:02,144 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:02,145 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:02,285 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:03,286 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:05,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:05,088 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:05,090 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:05,091 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:05,287 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:07,091 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:07,288 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:08,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:08,014 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:08,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:08,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:08,289 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:09,289 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:10,941 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:10,943 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:10,944 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:10,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:11,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:57:11,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:57:11,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:57:11,291 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:12,295 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:13,292 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:13,886 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:13,888 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:13,889 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:13,890 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:14,293 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:15,293 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:16,825 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:16,827 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:16,828 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:16,829 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:17,295 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:17,832 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:19,296 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:19,768 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:19,770 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:19,770 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:19,771 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:20,297 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:21,298 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:22,780 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:22,782 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:22,782 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:22,784 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:23,299 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:23,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:25,301 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:25,707 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:25,709 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:25,710 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:25,711 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:26,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:57:26,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:57:26,050 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:57:26,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:57:26,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:27,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:28,637 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:28,638 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:28,639 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:28,640 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:29,304 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:29,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:31,305 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:31,565 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:31,567 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:31,568 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:31,569 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:32,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:33,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:34,521 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:34,523 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:34,525 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:34,526 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:35,308 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:35,527 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:37,309 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:37,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:37,456 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:37,457 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:37,458 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:38,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:39,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:40,378 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:40,381 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:40,381 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:40,382 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:41,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:57:41,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:57:41,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:57:41,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:41,311 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:43,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:43,316 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:43,318 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:43,318 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:43,319 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:44,313 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:45,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:46,241 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:46,243 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:46,244 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:46,245 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:46,315 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:47,245 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:47,315 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:49,169 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:49,171 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:49,171 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:49,172 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:49,317 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:51,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:52,105 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:52,107 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:52,107 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:52,109 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:52,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:53,109 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:53,319 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:55,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:55,043 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:55,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:55,044 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:55,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:56,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:57:56,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:57:56,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:57:56,238 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:57:57,321 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:57:57,990 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:57:57,993 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:57:57,993 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:57:57,994 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:57:58,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:57:58,995 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:57:59,323 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:00,943 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:00,946 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:00,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:00,947 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:01,325 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:03,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:03,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:03,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:03,892 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:03,893 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:04,327 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:04,893 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:05,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:06,906 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:06,909 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:06,909 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:06,910 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:07,329 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:09,330 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:09,833 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:09,835 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:09,836 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:09,837 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:10,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:10,838 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:11,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:58:11,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:58:11,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:58:11,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:12,760 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:12,763 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:12,763 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:12,764 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:13,334 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:15,335 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:15,691 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:15,693 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:15,694 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:15,695 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:16,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:16,696 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:17,337 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:18,637 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:18,640 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:18,640 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:18,641 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:19,339 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:21,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:21,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:21,588 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:21,588 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:21,589 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:22,341 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:22,590 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:23,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:24,532 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:24,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:24,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:24,536 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:25,343 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:26,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:58:26,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:58:26,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:58:26,310 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:58:27,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:27,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:27,468 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:27,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:27,470 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:28,345 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:28,471 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:29,346 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:30,389 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:30,392 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:30,392 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:30,393 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:31,347 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:31,347 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:33,315 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:33,317 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:33,318 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:33,319 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:33,349 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:34,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:35,350 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:36,248 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:36,250 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:36,251 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:36,252 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:36,350 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:37,351 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:39,180 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:39,182 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:39,182 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:39,184 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:39,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:40,184 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:41,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:58:41,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:58:41,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:58:41,353 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:42,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:42,109 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:42,110 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:42,111 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:42,354 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:43,355 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:45,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:45,041 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:45,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:45,042 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:45,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:46,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:47,357 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:48,010 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:48,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:48,013 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:48,014 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:48,358 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:49,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:50,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:50,940 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:50,940 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:50,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:51,360 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:51,942 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:53,361 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:53,863 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:53,866 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:53,866 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:53,867 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:54,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:55,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:56,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:58:56,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:58:56,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:58:56,312 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:58:56,791 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:56,812 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:56,812 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:56,813 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:58:57,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:58:57,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:58:59,366 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:58:59,722 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:58:59,724 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:58:59,725 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:58:59,726 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:00,367 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:01,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:02,650 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:02,653 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:02,653 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:02,654 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:03,370 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:03,655 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:05,371 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:05,577 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:05,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:05,579 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:05,580 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:06,372 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:07,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:08,503 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:08,505 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:08,505 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:08,507 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:09,374 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:09,507 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:59:11,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:59:11,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:59:11,375 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:11,439 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:11,441 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:11,441 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:11,442 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:12,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:13,377 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:14,397 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:14,399 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:14,400 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:14,401 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:15,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:15,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:15,401 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:17,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:17,324 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:17,324 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:17,325 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:17,380 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:19,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:20,248 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:20,250 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:20,250 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:20,251 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:20,382 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:21,252 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:21,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:23,193 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:23,195 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:23,195 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:23,197 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:23,385 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:25,386 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:26,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:59:26,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:59:26,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:59:26,144 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:26,319 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:59:26,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:26,319 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:26,321 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:26,322 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:26,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:27,388 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:29,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:29,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:29,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:29,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:29,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:31,391 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:32,096 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:32,098 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:32,099 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:32,099 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:32,100 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:32,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:33,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:35,028 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:35,030 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:35,031 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:35,032 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:35,394 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:37,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:37,963 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:37,965 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:37,965 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:37,966 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:37,967 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:38,396 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:39,396 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:40,905 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:40,908 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:40,908 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:40,909 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:41,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:59:41,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:59:41,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:59:41,398 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:43,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:43,398 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:43,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:43,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:43,838 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:43,839 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:44,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:45,400 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:46,767 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:46,769 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:46,769 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:46,770 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:47,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:48,771 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:49,402 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:49,711 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:49,719 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:49,719 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:49,720 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:50,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:51,404 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:52,642 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:52,644 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:52,644 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:52,645 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:53,405 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:54,646 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 19:59:55,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:55,576 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:55,578 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:55,578 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:55,579 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:56,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 19:59:56,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 19:59:56,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 19:59:56,219 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 19:59:56,407 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 19:59:57,407 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:58,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 19:59:58,506 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 19:59:58,506 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 19:59:58,519 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 19:59:59,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 19:59:59,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:00,520 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:01,459 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:01,461 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:01,462 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:01,463 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:02,411 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:03,411 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:04,408 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:04,410 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:04,411 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:04,412 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:04,412 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:05,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:06,412 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:07,336 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:07,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:07,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:07,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:07,414 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:09,415 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:10,264 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:10,266 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:10,266 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:10,268 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:10,416 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:00:11,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:00:11,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:00:11,417 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:12,283 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:13,193 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:13,196 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:13,196 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:13,197 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:13,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:15,419 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:16,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:16,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:16,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:16,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:16,420 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:17,420 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:18,217 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:19,151 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:19,153 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:19,154 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:19,155 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:19,422 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:21,423 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:22,089 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:22,092 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:22,092 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:22,093 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:22,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:23,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:24,094 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:25,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:25,023 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:25,024 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:25,025 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:25,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:26,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:00:26,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:00:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:00:26,254 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:00:27,427 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:27,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:27,979 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:27,979 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:27,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:28,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:29,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:30,022 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:30,904 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:30,906 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:30,907 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:30,908 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:31,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:33,431 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:33,848 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:33,851 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:33,851 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:33,852 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:34,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:35,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:35,853 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:36,774 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:36,776 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:36,776 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:36,778 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:37,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:39,435 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:39,698 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:39,706 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:39,706 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:39,707 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:40,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:41,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:00:41,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:00:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:00:41,284 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:41,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:42,628 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:42,630 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:42,630 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:42,631 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:43,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:45,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:45,553 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:45,554 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:45,555 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:45,556 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:46,440 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:46,557 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:47,440 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:48,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:48,487 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:48,487 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:48,488 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:49,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:49,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:51,424 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:51,426 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:51,427 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:51,428 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:51,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:52,429 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:53,444 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:54,345 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:54,347 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:54,348 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:54,349 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:54,445 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:55,445 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:00:56,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:00:56,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:00:56,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:00:56,297 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:00:57,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:00:57,275 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:00:57,276 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:00:57,278 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:00:57,447 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:00:58,278 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:00:59,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:00,292 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:00,294 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:00,294 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:00,296 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:00,449 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:01,449 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:03,223 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:03,225 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:03,226 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:03,227 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:03,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:04,227 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:05,452 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:06,149 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:06,151 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:06,152 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:06,153 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:06,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:07,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:09,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:09,077 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:09,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:09,079 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:09,455 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:10,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:01:11,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:01:11,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:01:11,456 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:12,007 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:12,009 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:12,009 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:12,011 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:12,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:13,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:14,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:14,939 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:14,939 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:14,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:15,459 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:15,941 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:17,460 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:17,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:17,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:17,870 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:17,871 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:18,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:19,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:20,796 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:20,798 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:20,799 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:20,800 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:21,463 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:21,800 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:23,463 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:23,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:23,727 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:23,727 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:23,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:24,464 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:25,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:26,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:01:26,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:01:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:01:26,278 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:01:26,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:26,666 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:26,666 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:26,667 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:27,466 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:27,668 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:29,467 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:29,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:29,617 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:29,617 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:29,618 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:30,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:31,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:32,562 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:32,564 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:32,564 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:32,565 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:33,470 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:33,470 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:33,566 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:35,493 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:35,495 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:35,495 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:35,497 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:36,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:37,473 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:38,431 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:38,433 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:38,433 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:38,434 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:38,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:39,435 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:39,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:41,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:01:41,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:01:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:01:41,396 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:41,399 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:41,399 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:41,400 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:41,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:43,477 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:44,442 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:44,468 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:44,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:44,469 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:44,470 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:44,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:45,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:47,367 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:47,369 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:47,370 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:47,371 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:47,479 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:49,480 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:50,291 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:50,293 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:50,293 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:50,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:50,295 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:50,481 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:51,482 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:53,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:53,244 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:53,244 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:53,247 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:53,483 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:55,484 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:56,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:01:56,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:01:56,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:01:56,205 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:56,247 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:01:56,247 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:01:56,248 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:56,248 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:56,249 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:56,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:01:57,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:01:59,173 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:01:59,175 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:01:59,176 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:01:59,177 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:01:59,487 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:01,489 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:02,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:02,102 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:02,103 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:02,103 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:02,104 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:02,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:03,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:05,029 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:05,032 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:05,033 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:05,034 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:05,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:07,493 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:07,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:07,963 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:07,963 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:07,963 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:07,965 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:08,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:09,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:10,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:10,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:10,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:10,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:11,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:02:11,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:02:11,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:02:11,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:13,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:13,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:13,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:13,816 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:13,816 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:13,817 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:14,497 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:15,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:16,749 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:16,751 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:16,751 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:16,752 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:17,499 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:18,753 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:19,500 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:19,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:19,681 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:19,682 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:19,683 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:20,501 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:21,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:22,602 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:22,604 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:22,604 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:22,606 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:23,503 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:23,503 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:24,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:25,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:25,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:25,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:25,633 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:26,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:02:26,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:02:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:02:26,322 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:02:26,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:27,506 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:28,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:28,583 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:28,584 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:28,585 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:29,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:29,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:30,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:31,531 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:31,533 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:31,533 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:31,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:32,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:33,524 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:34,490 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:34,493 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:34,493 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:34,494 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:34,525 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:35,526 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:36,495 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:37,430 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:37,432 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:37,433 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:37,434 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:37,527 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:39,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:40,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:40,365 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:40,365 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:40,366 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:40,529 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:41,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:02:41,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:02:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:02:41,530 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:42,318 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:43,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:43,295 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:43,295 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:43,296 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:43,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:45,532 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:46,217 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:46,220 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:46,221 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:46,222 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:46,533 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:47,534 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:48,222 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:49,153 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:49,155 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:49,156 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:49,157 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:49,535 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:51,536 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:52,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:52,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:52,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:52,085 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:52,537 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:53,537 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:54,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:02:55,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:55,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:55,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:55,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:55,539 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:56,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:02:56,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:02:56,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:02:56,315 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:02:57,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:57,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:02:57,956 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:02:57,957 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:02:57,958 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:02:58,541 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:02:59,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:02:59,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:00,909 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:00,911 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:00,911 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:00,912 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:01,543 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:03,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:03,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:03,844 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:03,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:03,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:04,545 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:05,547 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:05,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:06,881 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:06,884 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:06,885 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:06,886 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:07,549 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:09,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:09,810 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:09,816 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:09,817 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:09,818 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:10,551 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:11,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:03:11,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:03:11,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:03:11,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:11,552 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:12,759 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:12,761 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:12,761 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:12,763 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:13,553 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:13,553 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:15,701 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:15,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:15,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:15,704 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:16,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:16,705 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:17,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:18,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:18,636 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:18,637 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:18,638 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:18,888 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:19,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:19,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:21,558 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:22,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:22,007 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:22,008 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:22,008 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:22,009 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:22,559 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:23,559 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:24,943 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:24,945 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:24,945 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:24,946 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:25,561 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:26,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:03:26,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:03:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:03:26,312 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:03:27,312 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:27,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:27,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:27,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:27,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:27,869 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:28,563 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:29,563 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:30,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:30,803 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:30,803 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:30,805 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:31,565 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:32,805 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:33,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:33,735 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:33,737 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:33,738 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:33,739 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:34,567 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:35,567 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:36,680 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:36,683 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:36,683 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:36,684 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:37,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:37,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:38,684 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:39,605 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:39,607 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:39,608 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:39,609 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:40,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:41,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:03:41,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:03:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:03:41,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:42,535 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:42,537 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:42,537 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:42,538 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:42,572 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:43,573 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:44,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:45,502 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:45,504 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:45,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:45,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:45,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:47,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:48,437 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:48,439 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:48,439 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:48,440 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:48,576 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:49,577 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:50,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:51,489 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:51,491 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:51,492 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:51,493 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:51,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:53,579 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:54,414 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:54,417 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:54,417 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:54,418 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:54,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:55,581 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:03:56,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:03:56,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:03:56,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:03:56,292 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:03:56,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:03:57,343 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:03:57,347 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:03:57,347 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:03:57,348 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:03:57,582 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:03:59,583 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:00,272 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:00,274 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:00,274 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:00,275 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:00,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:01,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:02,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:03,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:03,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:03,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:03,217 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:03,586 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:05,587 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:06,142 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:06,144 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:06,145 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:06,146 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:06,588 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:07,589 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:08,146 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:09,089 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:09,091 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:09,091 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:09,092 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:09,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:11,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:04:11,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:04:11,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:04:11,591 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:12,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:12,014 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:12,015 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:12,016 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:12,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:13,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:14,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:14,934 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:14,936 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:14,936 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:14,937 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:15,594 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:17,595 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:17,862 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:17,863 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:17,864 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:17,866 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:18,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:19,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:19,866 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:20,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:20,792 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:20,792 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:20,793 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:21,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:21,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:23,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:23,731 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:23,731 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:23,732 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:24,600 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:25,600 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:25,733 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:26,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:04:26,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:04:26,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:04:26,298 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:04:26,658 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:26,660 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:26,660 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:26,661 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:27,602 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:27,602 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:29,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:29,582 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:29,582 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:29,583 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:29,603 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:31,584 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:31,604 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:32,516 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:32,519 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:32,519 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:32,520 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:32,605 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:33,605 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:35,535 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:35,537 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:35,538 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:35,539 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:35,607 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:37,540 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:37,608 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:38,478 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:38,480 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:38,481 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:38,482 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:38,609 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:39,609 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:41,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:04:41,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:04:41,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:04:41,417 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:41,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:41,420 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:41,421 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:41,611 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:43,422 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:43,612 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:44,347 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:44,350 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:44,350 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:44,352 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:44,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:45,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:47,281 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:47,290 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:47,290 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:47,291 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:47,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:49,292 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:49,616 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:50,216 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:50,218 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:50,219 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:50,220 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:50,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:51,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:53,158 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:53,161 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:53,161 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:53,162 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:53,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:55,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:04:55,620 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:56,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:04:56,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:04:56,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:04:56,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:56,282 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:04:56,282 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:56,282 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:56,283 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:56,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:04:57,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:04:59,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:04:59,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:04:59,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:04:59,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:04:59,623 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:01,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:01,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:01,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:01,961 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:01,962 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:01,963 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:02,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:03,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:04,892 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:04,894 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:04,895 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:04,896 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:05,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:05,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:06,897 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:07,827 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:07,830 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:07,830 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:07,831 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:08,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:09,630 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:10,779 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:10,781 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:10,781 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:10,782 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:11,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:05:11,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:05:11,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:05:11,632 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:11,632 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:12,246 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:13,721 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:13,724 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:13,724 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:13,725 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:14,634 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:15,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:16,647 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:16,648 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:16,649 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:16,650 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:17,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:17,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:17,651 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:19,665 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:19,667 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:19,668 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:19,669 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:20,638 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:21,639 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:22,610 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:22,612 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:22,612 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:22,613 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:22,640 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:23,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:23,640 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:25,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:25,541 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:25,541 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:25,542 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:25,642 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:26,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:05:26,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:05:26,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:05:26,281 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:05:27,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:28,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:28,469 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:28,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:28,470 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:28,644 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:29,471 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:29,644 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:31,397 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:31,399 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:31,400 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:31,401 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:31,646 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:33,647 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:34,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:34,326 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:34,326 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:34,327 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:34,648 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:35,328 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:35,648 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:37,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:37,272 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:37,272 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:37,273 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:37,650 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:39,651 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:40,202 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:40,204 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:40,204 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:40,206 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:40,652 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:41,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:05:41,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:05:41,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:05:41,215 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:41,652 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:43,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:43,146 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:43,147 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:43,149 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:43,654 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:45,655 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:46,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:46,075 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:46,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:46,076 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:46,656 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:47,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:47,656 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:49,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:49,014 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:49,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:49,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:49,658 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:51,659 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:51,947 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:51,949 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:51,950 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:51,951 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:52,660 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:52,951 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:53,660 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:54,912 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:54,914 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:54,915 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:54,916 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:55,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:05:55,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:56,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:05:56,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:05:56,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:05:56,259 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:05:57,854 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:05:57,856 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:05:57,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:05:57,858 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:05:58,664 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:05:58,859 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:05:59,664 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:00,877 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:00,880 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:00,880 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:00,881 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:01,666 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:01,666 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:03,804 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:03,806 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:03,807 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:03,809 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:04,668 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:04,809 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:05,668 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:06,745 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:06,747 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:06,747 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:06,748 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:07,669 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:07,670 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:09,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:09,684 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:09,684 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:09,685 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:10,671 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:10,686 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:11,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:06:11,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:06:11,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:06:11,672 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:12,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:12,608 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:12,609 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:12,610 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:12,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:13,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:15,531 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:15,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:15,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:15,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:15,675 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:16,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:17,675 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:18,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:18,457 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:18,457 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:18,458 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:18,676 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:19,677 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:21,391 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:21,393 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:21,394 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:21,395 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:21,678 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:22,395 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:23,679 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:24,314 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:24,316 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:24,316 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:24,318 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:24,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:25,681 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:06:26,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:06:26,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:06:26,244 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:06:27,236 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:27,241 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:27,242 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:27,243 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:27,682 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:28,243 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:29,683 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:30,197 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:30,200 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:30,200 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:30,201 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:30,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:31,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:33,140 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:33,142 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:33,143 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:33,144 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:33,686 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:34,144 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:35,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:36,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:36,085 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:36,086 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:36,087 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:36,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:37,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:39,015 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:39,017 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:39,017 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:39,018 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:39,689 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:39,690 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:40,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:06:41,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:06:41,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:06:41,938 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:41,940 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:41,941 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:41,942 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:42,691 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:43,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:44,982 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:44,985 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:44,985 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:44,986 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:45,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:45,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:45,987 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:47,904 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:47,906 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:47,907 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:47,908 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:48,695 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:49,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:50,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:50,844 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:50,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:50,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:51,697 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:51,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:51,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:53,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:53,779 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:53,780 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:53,781 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:54,699 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:55,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:56,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:06:56,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:06:56,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:06:56,279 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:06:56,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:56,701 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:56,702 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:56,703 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:57,703 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:06:57,703 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:06:57,703 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:06:59,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:06:59,624 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:06:59,624 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:06:59,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:06:59,705 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:01,706 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:02,544 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:02,548 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:02,549 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:02,550 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:02,706 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:03,550 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:03,707 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:05,465 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:05,467 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:05,468 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:05,470 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:05,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:07,709 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:08,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:08,396 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:08,397 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:08,398 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:08,710 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:09,399 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:09,711 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:11,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:07:11,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:07:11,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:07:11,316 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:11,320 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:11,320 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:11,322 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:11,712 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:13,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:14,259 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:14,261 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:14,262 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:14,263 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:14,714 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:15,264 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:15,715 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:17,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:17,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:17,191 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:17,192 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:17,717 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:19,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:20,156 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:20,158 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:20,159 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:20,160 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:20,719 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:21,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:21,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:23,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:23,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:23,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:23,085 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:23,721 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:25,723 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:07:26,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:07:26,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:07:26,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:26,322 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:07:26,322 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:26,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:26,323 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:26,324 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:26,724 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:27,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:29,022 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:29,025 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:29,025 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:29,026 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:29,726 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:29,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:31,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:31,949 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:31,949 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:31,949 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:31,950 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:32,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:33,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:34,867 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:34,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:34,870 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:34,871 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:35,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:35,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:37,805 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:37,807 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:37,807 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:37,808 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:37,809 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:38,733 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:39,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:40,753 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:40,756 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:40,756 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:40,757 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:07:41,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:07:41,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:07:41,735 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:41,736 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:43,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:43,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:43,684 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:43,684 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:43,685 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:43,737 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:45,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:46,613 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:46,616 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:46,616 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:46,617 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:46,739 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:47,740 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:48,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:49,541 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:49,543 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:49,543 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:49,545 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:49,742 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:51,743 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:52,468 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:52,470 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:52,471 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:52,472 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:52,744 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:53,745 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:54,472 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:07:55,427 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:55,429 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:55,430 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:55,431 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:55,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:56,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:07:56,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:07:56,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:07:56,299 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:07:57,748 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:07:58,369 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:07:58,371 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:07:58,372 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:07:58,373 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:07:58,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:07:59,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:00,373 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:01,312 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:01,314 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:01,315 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:01,316 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:01,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:03,752 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:04,250 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:04,253 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:04,253 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:04,254 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:04,753 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:05,755 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:06,255 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:07,277 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:07,280 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:07,280 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:07,281 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:07,756 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:09,757 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:10,211 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:10,213 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:10,213 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:10,215 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:10,758 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:11,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:08:11,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:08:11,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:08:11,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:11,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:13,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:13,143 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:13,144 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:13,145 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:13,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:13,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:16,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:16,086 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:16,086 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:16,087 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:16,763 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:17,088 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:17,764 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:19,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:19,015 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:19,015 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:19,016 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:19,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:19,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:21,935 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:21,937 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:21,937 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:21,938 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:22,767 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:22,939 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:23,768 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:24,856 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:24,859 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:24,859 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:24,860 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:25,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:25,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:26,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:08:26,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:08:26,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:08:26,291 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:08:27,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:27,786 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:27,786 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:27,787 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:28,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:28,788 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:29,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:30,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:30,710 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:30,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:30,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:30,772 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:31,773 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:33,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:33,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:33,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:33,633 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:33,774 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:34,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:35,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:36,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:36,556 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:36,556 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:36,557 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:36,776 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:37,777 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:39,475 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:39,478 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:39,478 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:39,479 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:39,778 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:40,480 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:41,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:08:41,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:08:41,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:08:41,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:42,401 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:42,404 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:42,404 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:42,405 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:42,780 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:43,781 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:45,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:45,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:45,323 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:45,324 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:45,782 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:46,325 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:47,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:48,249 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:48,251 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:48,251 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:48,252 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:48,784 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:49,785 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:51,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:51,270 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:51,270 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:51,271 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:51,786 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:52,272 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:08:53,787 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:54,191 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:54,193 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:54,193 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:54,194 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:54,788 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:55,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:56,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:08:56,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:08:56,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:08:56,277 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:08:57,120 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:08:57,122 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:08:57,122 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:08:57,124 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:08:57,790 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:08:57,790 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:08:58,124 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:00,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:00,051 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:00,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:00,053 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:00,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:01,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:02,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:02,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:02,981 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:02,982 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:03,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:03,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:03,982 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:05,902 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:05,904 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:05,905 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:05,907 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:06,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:07,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:08,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:08,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:08,838 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:08,839 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:09,798 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:09,798 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:09,839 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:11,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:09:11,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:09:11,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:09:11,792 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:11,794 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:11,794 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:11,795 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:11,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:13,800 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:14,721 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:14,723 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:14,723 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:14,725 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:14,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:15,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:15,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:17,651 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:17,654 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:17,654 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:17,655 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:17,803 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:19,804 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:20,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:20,584 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:20,585 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:20,586 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:20,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:21,586 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:21,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:23,511 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:23,514 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:23,514 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:23,515 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:23,807 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:25,807 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:26,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:09:26,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:09:26,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:09:26,310 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:09:26,474 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:26,476 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:26,476 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:26,477 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:26,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:27,478 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:27,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:29,400 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:29,403 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:29,403 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:29,404 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:29,810 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:31,811 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:32,450 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:32,452 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:32,452 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:32,453 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:32,812 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:33,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:33,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:35,373 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:35,375 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:35,376 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:35,377 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:35,814 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:37,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:38,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:38,330 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:38,330 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:38,331 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:38,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:39,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:39,817 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:41,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:09:41,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:09:41,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:09:41,252 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:41,254 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:41,254 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:41,255 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:41,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:41,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:43,819 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:44,179 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:44,183 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:44,184 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:44,185 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:44,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:45,185 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:45,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:47,103 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:47,105 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:47,106 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:47,107 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:47,822 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:47,822 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:50,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:50,050 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:50,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:50,052 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:50,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:51,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:51,825 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:52,994 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:52,996 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:52,997 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:52,998 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:53,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:53,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:55,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:55,921 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:55,922 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:55,923 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:56,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:09:56,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:09:56,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:09:56,271 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:09:56,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:09:56,828 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:09:57,829 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:58,853 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:09:58,856 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:09:58,856 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:09:58,857 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:09:59,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:09:59,831 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:01,780 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:01,783 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:01,783 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:01,783 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:01,784 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:01,832 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:03,833 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:04,701 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:04,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:04,704 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:04,705 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:04,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:05,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:07,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:07,624 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:07,624 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:07,624 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:07,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:07,836 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:09,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:10,556 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:10,558 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:10,559 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:10,560 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:10,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:11,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:10:11,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:10:11,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:10:11,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:13,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:13,483 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:13,485 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:13,486 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:13,487 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:13,840 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:15,841 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:16,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:16,455 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:16,455 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:16,456 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:16,842 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:17,842 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:18,457 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:19,373 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:19,375 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:19,375 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:19,376 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:19,844 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:21,845 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:22,292 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:22,294 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:22,295 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:22,296 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:22,846 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:23,847 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:24,297 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:25,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:25,227 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:25,227 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:25,228 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:25,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:25,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:26,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:10:26,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:10:26,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:10:26,264 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:10:28,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:28,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:28,169 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:28,170 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:28,850 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:29,851 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:30,171 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:31,097 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:31,100 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:31,100 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:31,101 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:31,853 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:31,853 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:34,018 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:34,020 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:34,021 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:34,022 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:34,855 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:35,856 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:36,022 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:36,948 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:36,950 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:36,950 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:36,951 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:37,857 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:37,857 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:39,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:39,892 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:39,893 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:39,894 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:40,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:41,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:10:41,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:10:41,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:10:41,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:41,860 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:42,828 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:42,830 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:42,831 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:42,832 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:42,861 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:43,862 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:45,771 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:45,773 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:45,773 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:45,774 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:45,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:46,775 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:47,865 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:48,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:48,697 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:48,698 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:48,699 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:48,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:49,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:51,627 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:51,629 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:51,629 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:51,630 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:51,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:52,631 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:53,869 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:54,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:54,549 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:54,550 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:54,551 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:54,870 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:55,871 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:10:56,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:10:56,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:10:56,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:10:56,228 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:10:57,514 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:10:57,516 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:10:57,517 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:10:57,518 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:10:57,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:10:58,518 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:10:59,874 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:00,437 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:00,441 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:00,441 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:00,442 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:00,875 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:01,876 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:03,368 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:03,370 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:03,371 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:03,372 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:03,878 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:04,372 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:05,879 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:06,308 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:06,310 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:06,310 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:06,311 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:06,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:07,881 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:09,234 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:09,236 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:09,236 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:09,237 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:09,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:09,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:10,238 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:11,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:11:11,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:11:11,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:11:12,154 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:12,156 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:12,156 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:12,158 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:12,884 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:13,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:15,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:15,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:15,085 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:15,086 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:15,887 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:15,887 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:16,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:18,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:18,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:18,011 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:18,012 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:18,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:19,890 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:20,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:20,940 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:20,940 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:20,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:21,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:21,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:21,941 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:23,882 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:23,884 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:23,884 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:23,885 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:23,893 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:25,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:26,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:11:26,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:11:26,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:11:26,287 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:11:26,815 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:26,817 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:26,817 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:26,819 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:26,895 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:27,819 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:27,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:29,754 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:29,758 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:29,758 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:29,759 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:29,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:31,899 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:32,684 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:32,686 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:32,687 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:32,688 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:32,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:33,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:33,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:35,617 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:35,619 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:35,620 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:35,621 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:35,903 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:37,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:38,540 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:38,544 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:38,545 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:38,546 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:38,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:39,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:39,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:41,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:11:41,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:11:41,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:11:41,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:41,541 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:41,542 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:41,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:41,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:43,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:44,476 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:44,479 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:44,479 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:44,480 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:44,910 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:45,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:45,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:47,434 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:47,437 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:47,437 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:47,438 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:47,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:49,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:50,370 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:50,372 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:50,373 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:50,374 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:50,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:51,374 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:51,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:53,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:53,298 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:53,298 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:53,300 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:53,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:53,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:56,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:11:56,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:11:56,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:11:56,232 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:56,265 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:11:56,266 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:56,266 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:56,267 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:56,918 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:11:57,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:11:57,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:59,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:11:59,164 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:11:59,164 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:11:59,165 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:11:59,920 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:11:59,921 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:02,096 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:02,098 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:02,099 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:02,100 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:02,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:03,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:03,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:05,023 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:05,025 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:05,025 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:05,027 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:05,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:05,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:07,952 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:07,954 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:07,955 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:07,956 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:08,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:08,956 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:09,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:10,903 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:10,905 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:10,906 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:10,907 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:10,928 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:11,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:12:11,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:12:11,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:12:11,928 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:13,832 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:13,834 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:13,834 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:13,836 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:13,930 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:14,836 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:15,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:16,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:16,779 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:16,779 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:16,780 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:16,932 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:17,932 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:19,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:19,711 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:19,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:19,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:19,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:20,713 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:21,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:22,633 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:22,635 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:22,636 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:22,637 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:22,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:23,937 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:25,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:25,624 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:25,624 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:25,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:25,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:26,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:12:26,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:12:26,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:12:26,288 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:12:26,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:27,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:28,576 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:28,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:28,579 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:28,580 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:28,941 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:29,942 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:31,503 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:31,505 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:31,505 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:31,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:31,506 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:31,944 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:33,945 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:34,433 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:34,435 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:34,436 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:34,437 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:34,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:35,947 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:37,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:37,365 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:37,365 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:37,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:37,366 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:37,948 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:37,949 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:40,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:40,291 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:40,291 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:40,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:40,950 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:41,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:12:41,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:12:41,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:12:41,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:43,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:43,232 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:43,233 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:43,233 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:43,234 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:43,952 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:43,953 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:46,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:46,162 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:46,162 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:46,164 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:46,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:47,955 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:49,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:49,092 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:49,092 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:49,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:49,094 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:49,956 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:49,956 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:52,027 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:52,029 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:52,030 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:52,031 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:52,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:53,959 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:54,949 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:54,951 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:54,952 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:54,952 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:12:54,953 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:54,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:55,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:12:56,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:12:56,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:12:56,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:12:56,236 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:12:57,889 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:12:57,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:12:57,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:12:57,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:12:57,962 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:12:59,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:00,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:00,844 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:00,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:00,845 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:00,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:00,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:01,964 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:03,793 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:03,796 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:03,796 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:03,797 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:03,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:05,966 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:06,732 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:06,734 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:06,735 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:06,735 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:06,736 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:06,967 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:06,984 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:07,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:08,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:10,148 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:10,151 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:10,151 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:10,153 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:10,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:11,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:13:11,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:13:11,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:13:11,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:12,279 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:13,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:13,105 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:13,106 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:13,107 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:13,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:13,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:16,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:16,014 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:16,015 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:16,016 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:16,974 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:17,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:18,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:18,945 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:18,948 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:18,948 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:18,949 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:18,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:19,976 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:21,878 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:21,880 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:21,881 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:21,882 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:21,977 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:23,882 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:23,978 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:24,824 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:24,826 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:24,826 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:24,827 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:24,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:25,980 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:26,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:13:26,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:13:26,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:13:26,237 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:13:27,758 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:27,761 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:27,761 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:27,762 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:27,981 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:29,762 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:29,982 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:30,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:30,691 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:30,692 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:30,693 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:30,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:31,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:33,620 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:33,622 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:33,622 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:33,623 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:33,985 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:35,624 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:35,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:36,550 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:36,552 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:36,552 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:36,553 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:36,987 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:37,987 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:39,482 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:39,485 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:39,485 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:39,486 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:39,989 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:39,989 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:41,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:13:41,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:13:41,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:13:41,310 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:42,413 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:42,415 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:42,416 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:42,417 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:42,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:43,991 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:45,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:45,343 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:45,344 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:45,345 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:45,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:45,993 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:46,346 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:48,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:48,270 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:48,270 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:48,272 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:48,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:49,995 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:51,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:51,192 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:51,192 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:51,193 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:51,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:51,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:52,194 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:13:54,186 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:54,188 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:54,188 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:54,189 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:54,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:55,999 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:56,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:13:56,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:13:56,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:13:56,229 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:13:57,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:13:57,115 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:13:57,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:13:57,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:13:58,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:13:58,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:13:58,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:00,032 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:00,035 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:00,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:00,036 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:01,002 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:02,003 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:02,956 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:02,958 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:02,959 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:02,960 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:03,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:03,960 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:04,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:05,895 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:05,897 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:05,897 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:05,899 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:06,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:08,007 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:08,872 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:08,875 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:08,875 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:08,876 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:09,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:09,877 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:10,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:11,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:14:11,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:14:11,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:14:11,793 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:11,795 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:11,795 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:11,797 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:12,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:14,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:14,727 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:14,729 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:14,730 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:14,731 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:15,011 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:15,731 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:16,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:17,659 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:17,661 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:17,662 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:17,663 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:18,013 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:20,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:20,594 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:20,596 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:20,596 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:20,597 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:21,015 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:21,598 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:22,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:23,519 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:23,521 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:23,522 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:23,523 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:24,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:24,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:26,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:14:26,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:14:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:14:26,303 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:14:26,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:26,443 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:26,443 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:26,444 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:27,019 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:27,445 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:28,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:29,373 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:29,376 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:29,376 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:29,377 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:30,021 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:30,021 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:32,301 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:32,304 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:32,304 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:32,305 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:33,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:33,306 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:34,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:35,254 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:35,257 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:35,257 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:35,258 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:36,025 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:36,025 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:38,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:38,296 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:38,296 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:38,297 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:39,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:39,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:40,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:41,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:14:41,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:14:41,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:14:41,218 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:41,329 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:41,329 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:41,331 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:42,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:42,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:44,161 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:44,163 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:44,164 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:44,165 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:45,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:45,166 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:47,032 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:47,088 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:47,092 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:47,092 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:47,093 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:48,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:49,034 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:50,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:50,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:50,011 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:50,012 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:50,035 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:51,013 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:52,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:52,931 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:52,932 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:52,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:53,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:53,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:55,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:55,848 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:55,850 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:55,851 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:55,852 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:56,038 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:14:56,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:14:56,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:14:56,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:14:56,313 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:14:56,313 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:14:58,769 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:14:58,771 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:14:58,772 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:14:58,773 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:14:59,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:14:59,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:01,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:01,690 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:01,692 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:01,693 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:01,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:01,694 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:02,042 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:04,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:04,612 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:04,614 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:04,615 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:04,616 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:05,059 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:06,060 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:07,533 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:07,535 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:07,536 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:07,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:07,537 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:08,062 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:08,062 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:10,455 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:10,457 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:10,458 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:10,459 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:11,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:15:11,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:15:11,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:15:11,064 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:13,064 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:13,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:13,384 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:13,386 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:13,387 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:13,388 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:14,065 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:15,066 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:16,316 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:16,319 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:16,319 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:16,321 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:17,067 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:18,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:19,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:19,247 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:19,250 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:19,250 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:19,251 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:20,069 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:21,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:22,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:22,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:22,292 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:22,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:23,071 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:24,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:25,073 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:25,209 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:25,212 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:25,212 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:25,213 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:26,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:15:26,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:15:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:15:26,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:26,233 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:15:27,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:28,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:28,138 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:28,139 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:28,140 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:29,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:30,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:31,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:31,075 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:31,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:31,077 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:31,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:31,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:33,079 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:33,999 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:34,001 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:34,001 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:34,002 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:34,080 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:36,003 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:36,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:36,932 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:36,932 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:36,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:37,082 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:37,083 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:39,084 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:39,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:39,857 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:39,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:39,859 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:40,085 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:41,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:15:41,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:15:41,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:15:41,301 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:42,798 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:42,800 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:42,800 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:42,801 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:43,087 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:43,087 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:45,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:45,723 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:45,725 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:45,726 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:45,727 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:46,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:46,727 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:48,644 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:48,647 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:48,647 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:48,648 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:49,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:49,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:51,093 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:51,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:51,576 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:51,576 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:51,578 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:52,094 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:52,578 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:53,095 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:54,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:54,507 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:54,507 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:54,509 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:55,097 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:56,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:15:56,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:15:56,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:15:56,301 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:15:57,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:15:57,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:15:57,455 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:15:57,455 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:15:57,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:15:58,099 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:15:58,457 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:15:59,100 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:00,418 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:00,420 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:00,420 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:00,422 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:01,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:03,103 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:03,336 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:03,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:03,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:03,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:04,104 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:04,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:05,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:06,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:06,366 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:06,366 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:06,367 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:07,107 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:09,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:09,295 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:09,298 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:09,298 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:09,299 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:10,109 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:10,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:11,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:16:11,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:16:11,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:16:11,110 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:12,246 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:12,248 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:12,249 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:12,250 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:13,111 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:15,113 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:15,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:15,180 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:15,180 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:15,181 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:16,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:16,182 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:17,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:18,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:18,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:18,114 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:18,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:18,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:21,037 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:21,039 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:21,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:21,041 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:21,119 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:21,119 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:22,041 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:23,120 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:23,979 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:23,982 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:23,982 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:23,984 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:24,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:26,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:16:26,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:16:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:16:26,262 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:16:26,905 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:26,908 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:26,908 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:26,909 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:27,123 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:27,124 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:27,910 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:29,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:29,839 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:29,841 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:29,841 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:29,842 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:30,126 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:32,768 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:32,770 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:32,771 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:32,772 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:33,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:33,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:33,772 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:35,129 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:35,698 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:35,701 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:35,701 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:35,702 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:36,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:37,131 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:38,649 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:38,651 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:38,651 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:38,652 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:39,133 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:39,653 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:41,053 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:16:41,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:16:41,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:16:41,134 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:41,596 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:41,598 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:41,598 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:41,599 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:42,135 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:43,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:44,550 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:44,552 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:44,553 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:44,554 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:45,137 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:45,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:47,139 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:47,491 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:47,493 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:47,494 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:47,495 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:48,140 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:49,141 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:50,512 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:50,515 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:50,515 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:50,516 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:51,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:51,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:53,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:53,477 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:53,479 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:53,480 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:53,481 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:54,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:55,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:56,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:16:56,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:16:56,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:16:56,305 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:16:56,409 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:56,411 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:56,411 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:56,412 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:16:57,147 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:16:57,413 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:16:59,148 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:16:59,344 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:16:59,346 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:16:59,347 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:16:59,348 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:00,149 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:01,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:02,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:02,270 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:02,270 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:02,272 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:03,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:03,272 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:05,153 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:05,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:05,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:05,191 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:05,192 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:06,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:07,155 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:08,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:08,120 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:08,120 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:08,121 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:08,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:09,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:11,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:11,049 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:11,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:11,050 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:11,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:17:11,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:17:11,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:17:11,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:11,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:13,159 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:13,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:13,973 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:13,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:13,974 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:14,161 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:14,975 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:16,916 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:16,918 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:16,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:16,920 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:17,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:17,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:19,164 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:19,840 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:19,842 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:19,843 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:19,844 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:20,165 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:20,844 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:22,768 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:22,770 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:22,770 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:22,772 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:23,167 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:23,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:25,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:25,700 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:25,702 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:25,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:25,704 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:26,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:17:26,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:17:26,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:17:26,170 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:26,251 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:17:26,251 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:27,170 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:28,730 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:28,733 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:28,733 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:28,734 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:29,172 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:31,173 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:31,658 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:31,660 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:31,661 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:31,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:31,662 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:32,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:33,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:34,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:34,585 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:34,585 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:34,586 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:35,177 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:37,178 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:37,541 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:37,543 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:37,544 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:37,544 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:37,545 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:38,179 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:39,180 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:40,467 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:40,470 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:40,470 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:40,471 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:41,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:17:41,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:17:41,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:17:41,181 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:43,182 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:43,274 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:43,397 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:43,399 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:43,399 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:43,400 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:44,184 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:45,184 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:46,323 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:46,325 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:46,325 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:46,326 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:47,186 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:48,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:49,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:49,248 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:49,250 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:49,251 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:49,252 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:50,188 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:51,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:52,206 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:52,209 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:52,209 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:52,210 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:53,191 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:54,211 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:17:55,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:55,135 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:55,136 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:55,137 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:55,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:55,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:17:56,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:17:56,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:17:56,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:17:56,222 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:17:57,194 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:17:58,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:17:58,054 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:17:58,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:17:58,056 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:17:58,195 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:00,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:00,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:00,972 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:00,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:00,974 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:01,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:01,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:03,199 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:03,899 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:03,901 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:03,901 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:03,903 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:04,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:05,903 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:06,824 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:06,826 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:06,827 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:06,828 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:07,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:07,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:09,203 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:09,756 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:09,758 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:09,759 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:09,760 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:10,205 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:11,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:18:11,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:18:11,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:18:11,205 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:11,309 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:12,781 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:12,784 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:12,784 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:12,785 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:13,207 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:15,208 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:15,712 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:15,714 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:15,715 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:15,716 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:16,209 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:16,716 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:17,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:18,642 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:18,644 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:18,644 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:18,646 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:19,212 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:21,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:21,570 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:21,572 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:21,572 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:21,573 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:22,214 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:22,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:23,215 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:24,499 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:24,501 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:24,501 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:24,502 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:25,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:26,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:18:26,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:18:26,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:18:26,268 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:18:27,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:27,428 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:27,430 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:27,430 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:27,431 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:28,219 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:28,432 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:29,219 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:30,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:30,366 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:30,366 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:30,367 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:31,221 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:33,222 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:33,307 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:33,309 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:33,310 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:33,311 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:34,224 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:34,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:35,224 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:36,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:36,262 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:36,263 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:36,264 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:37,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:39,188 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:39,190 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:39,191 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:39,192 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:39,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:39,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:40,192 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:41,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:18:41,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:18:41,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:18:41,229 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:42,116 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:42,118 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:42,119 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:42,120 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:42,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:45,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:45,063 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:45,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:45,064 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:45,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:45,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:46,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:47,233 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:47,989 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:47,991 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:47,991 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:47,992 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:48,235 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:50,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:50,932 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:50,932 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:50,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:51,237 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:51,237 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:51,933 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:53,238 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:53,878 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:53,880 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:53,881 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:53,882 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:54,239 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:55,240 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:56,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:18:56,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:18:56,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:18:56,285 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:18:56,912 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:56,915 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:56,915 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:56,916 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:18:57,242 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:18:57,917 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:18:59,243 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:18:59,839 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:18:59,841 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:18:59,842 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:18:59,843 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:00,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:01,245 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:02,766 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:02,769 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:02,769 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:02,784 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:03,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:03,785 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:05,248 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:05,714 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:05,739 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:05,740 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:05,741 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:06,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:07,250 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:08,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:08,663 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:08,664 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:08,665 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:09,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:09,665 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:11,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:19:11,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:19:11,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:19:11,252 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:11,600 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:11,603 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:11,603 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:11,604 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:12,254 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:13,254 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:14,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:14,542 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:14,542 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:14,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:15,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:15,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:17,257 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:17,479 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:17,481 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:17,482 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:17,483 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:18,258 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:19,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:20,421 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:20,423 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:20,424 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:20,432 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:21,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:21,432 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:23,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:23,349 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:23,351 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:23,351 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:23,353 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:24,262 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:25,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:19:26,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:19:26,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:19:26,272 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:26,303 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:19:26,303 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:26,303 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:26,305 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:27,265 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:27,305 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:29,218 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:29,220 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:29,220 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:29,221 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:29,266 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:29,266 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:31,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:32,156 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:32,159 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:32,159 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:32,160 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:32,268 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:33,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:35,095 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:35,097 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:35,098 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:35,099 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:35,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:35,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:37,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:38,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:38,109 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:38,110 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:38,111 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:38,272 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:39,111 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:41,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:41,038 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:41,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:41,040 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:41,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:19:41,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:19:41,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:19:41,274 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:41,274 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:43,275 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:43,955 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:43,957 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:43,958 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:43,959 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:44,276 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:44,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:45,277 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:46,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:46,896 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:46,897 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:46,898 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:47,278 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:49,279 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:49,827 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:49,830 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:49,831 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:49,832 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:50,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:50,832 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:51,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:52,781 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:52,783 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:52,784 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:52,785 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:53,282 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:55,283 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:55,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:55,701 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:55,702 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:55,703 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:56,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:19:56,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:19:56,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:19:56,275 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:19:56,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:19:56,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:19:57,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:19:58,625 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:19:58,626 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:19:58,627 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:19:58,628 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:19:59,286 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:01,287 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:01,551 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:01,554 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:01,554 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:01,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:01,555 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:02,288 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:03,288 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:04,484 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:04,486 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:04,486 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:04,487 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:05,290 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:07,291 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:07,407 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:07,410 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:07,410 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:07,410 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:07,411 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:08,292 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:09,292 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:10,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:10,342 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:10,342 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:10,343 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:11,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:20:11,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:20:11,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:20:11,294 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:13,244 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:13,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:13,296 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:13,297 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:13,298 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:13,298 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:13,298 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:15,299 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:16,214 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:16,216 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:16,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:16,217 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:16,300 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:19,140 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:19,142 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:19,143 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:19,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:19,144 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:19,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:19,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:21,303 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:22,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:22,155 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:22,155 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:22,156 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:22,304 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:24,157 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:25,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:25,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:25,085 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:25,086 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:25,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:25,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:20:26,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:20:26,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:20:26,255 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:20:27,307 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:28,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:28,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:28,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:28,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:28,308 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:29,308 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:30,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:30,938 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:30,940 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:30,941 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:30,942 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:31,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:33,311 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:33,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:33,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:33,868 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:33,869 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:34,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:35,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:35,869 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:36,812 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:36,814 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:36,814 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:36,816 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:37,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:39,315 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:39,738 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:39,742 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:39,742 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:39,743 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:40,316 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:41,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:20:41,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:20:41,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:20:41,317 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:41,643 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:42,670 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:42,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:42,672 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:42,673 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:43,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:45,319 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:45,588 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:45,591 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:45,591 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:45,592 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:46,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:47,321 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:47,593 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:48,507 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:48,509 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:48,510 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:48,511 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:49,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:51,323 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:51,450 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:51,452 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:51,452 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:51,453 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:52,324 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:53,324 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:53,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:54,391 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:54,393 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:54,394 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:54,395 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:55,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:56,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:20:56,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:20:56,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:20:56,249 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:20:57,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:20:57,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:20:57,323 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:20:57,324 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:20:57,327 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:20:57,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:20:59,325 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:20:59,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:00,272 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:00,274 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:00,275 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:00,276 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:00,329 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:03,262 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:03,265 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:03,265 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:03,266 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:03,331 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:03,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:05,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:05,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:06,194 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:06,196 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:06,197 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:06,198 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:06,333 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:09,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:09,135 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:09,135 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:09,136 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:09,335 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:09,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:11,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:21:11,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:21:11,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:21:11,303 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:11,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:12,098 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:12,101 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:12,101 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:12,102 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:12,337 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:13,338 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:15,025 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:15,027 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:15,028 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:15,029 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:15,339 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:17,030 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:17,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:17,987 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:17,989 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:17,989 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:17,990 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:18,341 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:19,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:20,927 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:20,929 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:20,929 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:20,930 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:21,343 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:22,931 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:23,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:23,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:23,857 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:23,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:23,859 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:24,345 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:25,345 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:21:26,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:21:26,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:21:26,323 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:21:26,782 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:26,784 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:26,784 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:26,785 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:27,347 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:28,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:29,348 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:29,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:29,710 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:29,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:29,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:30,349 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:31,349 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:32,655 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:32,657 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:32,657 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:32,658 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:33,351 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:34,659 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:35,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:35,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:35,608 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:35,608 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:35,609 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:36,353 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:37,353 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:38,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:38,545 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:38,545 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:38,546 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:39,355 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:40,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:41,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:21:41,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:21:41,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:21:41,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:41,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:41,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:41,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:41,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:42,357 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:43,357 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:44,446 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:44,448 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:44,449 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:44,450 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:45,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:46,450 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:47,360 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:47,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:47,422 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:47,422 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:47,424 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:48,361 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:49,361 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:50,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:50,355 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:50,355 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:50,356 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:50,362 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:52,357 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:53,282 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:53,284 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:53,284 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:53,285 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:53,364 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:53,364 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:55,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:56,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:21:56,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:21:56,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:21:56,209 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:56,317 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:21:56,317 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:56,318 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:56,319 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:56,366 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:21:58,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:21:59,137 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:21:59,139 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:21:59,140 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:21:59,141 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:21:59,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:21:59,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:01,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:02,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:02,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:02,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:02,085 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:02,370 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:03,370 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:04,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:05,033 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:05,036 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:05,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:05,037 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:05,372 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:07,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:07,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:07,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:07,981 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:07,982 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:08,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:09,374 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:09,982 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:10,907 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:10,910 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:10,910 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:10,911 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:11,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:22:11,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:22:11,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:22:11,375 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:13,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:13,838 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:13,840 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:13,840 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:13,841 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:14,377 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:15,378 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:15,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:16,773 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:16,776 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:16,776 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:16,777 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:17,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:19,380 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:19,716 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:19,720 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:19,720 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:19,721 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:20,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:21,382 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:21,722 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:22,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:22,699 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:22,699 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:22,701 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:23,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:25,384 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:25,626 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:25,628 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:25,628 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:25,629 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:26,055 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:22:26,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:22:26,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:22:26,307 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:22:26,385 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:27,307 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:27,386 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:28,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:28,576 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:28,577 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:28,580 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:29,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:31,388 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:31,545 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:31,547 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:31,547 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:31,549 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:32,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:32,549 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:33,390 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:34,474 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:34,476 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:34,477 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:34,478 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:35,391 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:37,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:37,405 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:37,407 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:37,407 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:37,408 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:38,394 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:38,409 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:39,394 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:40,346 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:40,348 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:40,349 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:40,350 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:40,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:41,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:22:41,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:22:41,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:22:43,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:43,275 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:43,276 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:43,277 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:43,397 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:43,397 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:44,277 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:45,398 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:46,236 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:46,239 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:46,239 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:46,240 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:46,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:49,184 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:49,187 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:49,188 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:49,189 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:49,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:49,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:50,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:51,402 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:52,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:52,135 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:52,135 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:52,136 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:52,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:53,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:55,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:55,077 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:55,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:55,079 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:55,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:22:55,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:55,405 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:22:56,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:22:56,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:22:56,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:22:56,268 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:22:57,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:22:58,437 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:22:58,439 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:22:58,439 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:22:58,440 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:22:59,407 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:00,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:01,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:01,367 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:01,368 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:01,369 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:01,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:01,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:03,410 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:04,314 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:04,317 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:04,317 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:04,318 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:04,411 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:06,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:07,244 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:07,246 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:07,246 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:07,247 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:07,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:07,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:09,414 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:10,174 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:10,176 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:10,177 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:10,178 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:10,415 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:11,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:23:11,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:23:11,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:23:11,416 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:12,302 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:13,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:13,123 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:13,123 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:13,125 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:13,417 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:15,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:16,142 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:16,145 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:16,145 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:16,146 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:16,419 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:17,420 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:18,147 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:19,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:19,080 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:19,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:19,082 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:19,421 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:21,422 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:22,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:22,037 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:22,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:22,038 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:22,423 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:23,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:24,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:24,983 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:24,985 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:24,986 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:24,987 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:25,425 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:26,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:23:26,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:23:26,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:23:26,330 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:23:27,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:27,934 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:27,936 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:27,936 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:27,937 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:28,427 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:29,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:29,938 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:30,867 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:30,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:30,870 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:30,872 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:31,429 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:33,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:33,802 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:33,804 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:33,804 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:33,806 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:34,431 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:35,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:35,806 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:36,743 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:36,745 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:36,746 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:36,747 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:37,433 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:39,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:39,687 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:39,690 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:39,690 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:39,691 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:40,435 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:41,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:23:41,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:23:41,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:23:41,287 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:41,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:42,613 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:42,615 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:42,616 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:42,617 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:43,437 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:45,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:45,535 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:45,538 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:45,538 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:45,539 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:46,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:46,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:47,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:48,470 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:48,472 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:48,473 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:48,474 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:49,441 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:51,402 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:51,405 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:51,405 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:51,406 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:51,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:51,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:52,407 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:53,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:54,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:54,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:54,324 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:54,325 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:54,444 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:56,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:23:56,056 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:23:56,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:23:56,235 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:23:57,262 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:23:57,264 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:23:57,264 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:23:57,265 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:23:57,446 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:23:57,446 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:23:58,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:23:59,447 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:00,291 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:00,294 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:00,294 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:00,295 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:00,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:01,449 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:03,224 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:03,227 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:03,227 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:03,229 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:03,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:04,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:05,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:06,154 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:06,156 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:06,157 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:06,158 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:06,452 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:07,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:09,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:09,083 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:09,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:09,085 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:09,454 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:10,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:11,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:24:11,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:24:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:24:11,455 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:12,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:12,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:12,013 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:12,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:12,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:13,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:14,939 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:14,941 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:14,942 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:14,943 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:15,459 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:15,944 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:17,459 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:17,866 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:17,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:17,868 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:17,869 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:18,460 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:19,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:20,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:20,797 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:20,797 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:20,799 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:21,462 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:21,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:23,463 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:23,714 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:23,716 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:23,716 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:23,718 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:24,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:25,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:26,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:24:26,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:24:26,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:24:26,317 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:24:26,635 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:26,637 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:26,637 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:26,639 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:27,467 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:27,639 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:29,467 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:29,559 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:29,561 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:29,562 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:29,563 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:30,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:31,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:32,490 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:32,492 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:32,493 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:32,494 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:33,470 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:33,494 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:35,419 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:35,421 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:35,421 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:35,422 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:35,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:35,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:37,473 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:38,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:38,355 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:38,355 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:38,357 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:38,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:39,357 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:39,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:41,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:24:41,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:24:41,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:24:41,282 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:41,333 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:41,333 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:41,334 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:41,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:43,477 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:44,305 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:44,308 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:44,308 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:44,309 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:44,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:45,310 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:45,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:47,238 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:47,241 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:47,242 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:47,243 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:47,480 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:49,481 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:50,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:50,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:50,170 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:50,171 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:50,482 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:51,171 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:51,482 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:53,095 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:53,121 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:53,122 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:53,123 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:53,484 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:55,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:56,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:56,038 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:56,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:56,040 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:56,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:24:56,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:24:56,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:24:56,232 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:24:56,232 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:24:56,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:24:57,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:24:58,967 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:24:58,969 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:24:58,969 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:24:58,970 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:24:59,488 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:01,489 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:01,899 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:01,901 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:01,902 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:01,902 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:01,903 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:02,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:03,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:04,829 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:04,831 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:04,832 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:04,834 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:05,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:07,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:07,756 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:07,758 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:07,759 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:07,759 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:07,760 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:08,493 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:09,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:10,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:10,681 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:10,682 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:10,683 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:11,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:25:11,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:25:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:25:11,495 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:13,333 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:13,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:13,627 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:13,629 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:13,630 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:13,631 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:14,497 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:15,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:16,575 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:16,577 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:16,577 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:16,578 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:17,499 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:18,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:19,497 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:19,500 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:19,500 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:19,501 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:19,501 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:20,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:21,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:22,423 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:22,426 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:22,426 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:22,427 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:22,503 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:23,504 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:24,428 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:25,436 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:25,438 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:25,439 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:25,440 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:25,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:26,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:25:26,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:25:26,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:25:26,229 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:25:27,506 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:28,391 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:28,393 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:28,394 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:28,395 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:28,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:29,508 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:30,396 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:31,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:31,344 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:31,345 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:31,346 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:31,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:33,510 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:34,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:34,272 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:34,273 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:34,274 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:34,511 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:35,512 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:36,274 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:37,199 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:37,201 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:37,202 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:37,203 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:37,513 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:39,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:40,146 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:40,148 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:40,149 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:40,150 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:40,515 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:41,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:25:41,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:25:41,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:25:41,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:41,516 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:43,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:43,074 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:43,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:43,088 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:43,517 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:45,518 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:46,010 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:46,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:46,013 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:46,014 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:46,519 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:47,015 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:47,520 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:48,933 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:48,936 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:48,936 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:48,937 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:49,521 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:51,522 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:51,873 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:51,875 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:51,876 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:51,877 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:52,523 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:52,877 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:53,524 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:54,818 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:54,821 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:54,821 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:54,822 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:55,525 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:56,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:25:56,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:25:56,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:25:56,250 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:25:57,526 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:25:57,744 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:25:57,746 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:25:57,746 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:25:57,747 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:25:58,527 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:25:58,748 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:25:59,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:00,672 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:00,674 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:00,675 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:00,676 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:01,529 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:03,530 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:03,630 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:03,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:03,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:03,634 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:04,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:04,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:05,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:06,560 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:06,562 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:06,563 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:06,564 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:07,533 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:09,534 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:09,605 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:09,607 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:09,608 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:09,609 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:10,535 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:10,609 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:11,057 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:26:11,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:26:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:26:11,536 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:12,534 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:12,537 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:12,537 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:12,538 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:13,539 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:15,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:15,487 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:15,487 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:15,489 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:15,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:15,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:16,489 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:17,541 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:18,415 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:18,417 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:18,418 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:18,419 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:18,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:19,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:21,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:21,350 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:21,350 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:21,352 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:21,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:22,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:23,545 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:24,291 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:24,293 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:24,294 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:24,295 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:24,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:25,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:26,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:26:26,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:26:26,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:26:26,231 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:26:27,223 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:27,225 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:27,225 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:27,226 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:27,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:28,227 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:29,549 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:30,166 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:30,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:30,169 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:30,170 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:30,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:31,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:33,095 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:33,098 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:33,098 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:33,099 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:33,552 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:34,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:35,553 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:36,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:36,048 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:36,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:36,049 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:36,554 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:37,554 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:38,976 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:38,978 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:38,978 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:38,982 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:39,556 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:39,983 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:41,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:26:41,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:26:41,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:26:41,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:41,924 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:41,926 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:41,926 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:41,927 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:42,558 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:43,558 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:44,851 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:44,853 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:44,853 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:44,854 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:45,560 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:45,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:47,561 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:47,776 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:47,778 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:47,779 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:47,780 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:48,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:49,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:50,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:50,797 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:50,797 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:50,798 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:51,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:51,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:53,565 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:53,749 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:53,751 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:53,751 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:53,752 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:54,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:55,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:56,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:26:56,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:26:56,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:26:56,328 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:26:56,680 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:56,682 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:56,683 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:56,684 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:26:57,568 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:26:57,684 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:26:59,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:26:59,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:26:59,636 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:26:59,637 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:26:59,638 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:00,570 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:01,570 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:02,564 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:02,566 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:02,567 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:02,568 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:02,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:03,569 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:05,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:05,490 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:05,490 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:05,491 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:05,573 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:05,573 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:07,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:08,411 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:08,413 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:08,413 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:08,415 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:08,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:09,415 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:09,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:11,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:27:11,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:27:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:27:11,337 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:11,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:11,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:11,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:11,577 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:13,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:14,265 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:14,267 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:14,267 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:14,268 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:14,579 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:15,269 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:15,579 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:17,237 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:17,241 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:17,241 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:17,242 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:17,581 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:19,582 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:20,168 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:20,171 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:20,171 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:20,173 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:20,583 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:21,173 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:21,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:23,118 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:23,121 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:23,121 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:23,122 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:23,588 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:25,589 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:26,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:26,049 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:26,049 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:26,050 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:26,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:27:26,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:27:26,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:27:26,224 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:27:26,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:26,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:27,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:28,984 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:28,986 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:28,987 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:28,988 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:29,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:31,593 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:31,989 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:35,155 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:35,158 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:35,158 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:35,159 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:35,595 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:37,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:37,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:38,377 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:38,379 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:38,379 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:38,380 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:38,597 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:39,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:41,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:27:41,058 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:27:41,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:27:41,284 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:41,287 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:41,287 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:41,288 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:41,600 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:42,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:43,601 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:44,191 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:44,192 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:44,192 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:44,193 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:44,602 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:45,602 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:47,106 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:47,108 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:47,108 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:47,109 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:47,604 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:48,110 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:49,605 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:50,013 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:50,015 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:50,016 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:50,017 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:50,605 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:51,606 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:52,927 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:52,930 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:52,930 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:52,931 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:53,607 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:53,932 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:27:55,608 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:55,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:55,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:55,838 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:55,839 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:56,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:27:56,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:27:56,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:27:56,255 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:27:56,609 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:57,610 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:27:58,749 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:27:58,752 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:27:58,752 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:27:58,753 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:27:59,612 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:27:59,754 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:01,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:01,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:01,667 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:01,667 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:01,668 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:02,614 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:03,614 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:04,570 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:04,573 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:04,573 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:04,574 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:04,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:05,575 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:05,616 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:07,477 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:07,480 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:07,480 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:07,481 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:07,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:09,618 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:10,384 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:10,387 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:10,388 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:10,389 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:10,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:28:11,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:28:11,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:28:11,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:11,620 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:13,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:13,300 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:13,301 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:13,302 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:13,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:15,622 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:16,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:16,210 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:16,211 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:16,212 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:16,623 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:17,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:17,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:19,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:19,110 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:19,110 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:19,111 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:19,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:21,626 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:22,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:22,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:22,012 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:22,013 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:22,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:23,014 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:23,628 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:24,910 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:24,912 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:24,913 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:24,914 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:25,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:26,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:28:26,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:28:26,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:28:26,335 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:28:27,630 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:27,810 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:27,812 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:27,813 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:27,814 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:28,631 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:28,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:29,632 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:30,712 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:30,714 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:30,714 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:30,715 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:31,634 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:33,612 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:33,618 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:33,618 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:33,620 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:33,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:33,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:34,620 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:35,637 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:36,566 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:36,569 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:36,569 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:36,570 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:36,638 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:37,639 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:39,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:39,468 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:39,468 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:39,469 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:39,641 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:40,470 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:41,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:28:41,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:28:41,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:28:41,642 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:42,366 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:42,369 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:42,369 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:42,370 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:42,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:43,644 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:45,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:45,268 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:45,268 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:45,270 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:45,645 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:46,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:47,646 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:48,165 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:48,168 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:48,168 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:48,169 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:48,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:49,681 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:51,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:51,072 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:51,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:51,073 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:51,683 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:52,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:53,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:53,969 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:53,971 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:53,971 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:53,972 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:54,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:55,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:28:56,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:28:56,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:28:56,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:28:56,315 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:28:56,887 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:28:56,889 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:28:56,890 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:28:56,891 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:28:57,691 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:28:57,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:28:59,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:00,934 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:00,938 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:00,938 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:00,940 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:01,693 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:02,940 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:03,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:03,854 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:03,856 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:03,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:03,858 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:04,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:05,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:06,751 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:06,753 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:06,753 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:06,754 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:07,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:08,755 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:09,658 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:09,660 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:09,661 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:09,662 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:09,699 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:09,699 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:29:11,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:29:11,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:29:11,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:12,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:12,710 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:12,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:12,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:13,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:13,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:14,713 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:15,617 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:15,619 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:15,619 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:15,621 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:15,703 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:17,704 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:18,518 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:18,520 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:18,520 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:18,521 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:18,705 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:19,705 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:20,522 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:21,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:21,422 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:21,423 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:21,424 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:21,707 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:23,707 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:24,343 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:24,345 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:24,345 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:24,346 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:24,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:25,709 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:26,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:29:26,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:29:26,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:29:26,329 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:29:26,329 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:27,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:27,262 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:27,262 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:27,263 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:27,710 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:29,711 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:30,161 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:30,163 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:30,164 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:30,165 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:30,712 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:31,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:32,165 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:33,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:33,064 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:33,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:33,065 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:33,714 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:35,715 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:35,962 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:35,965 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:35,965 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:35,966 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:36,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:37,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:37,966 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:38,861 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:38,863 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:38,864 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:38,865 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:39,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:41,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:29:41,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:29:41,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:29:41,719 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:41,762 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:41,764 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:41,764 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:41,766 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:42,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:43,721 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:43,766 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:44,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:44,664 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:44,664 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:44,665 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:44,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:45,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:47,555 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:47,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:47,558 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:47,559 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:47,724 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:49,560 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:49,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:50,449 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:50,451 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:50,452 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:50,453 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:50,726 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:51,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:53,343 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:53,346 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:53,346 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:53,347 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:53,728 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:55,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:29:55,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:56,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:29:56,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:29:56,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:29:56,237 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:56,269 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:29:56,269 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:56,270 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:56,271 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:56,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:29:57,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:29:59,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:29:59,134 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:29:59,134 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:29:59,135 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:29:59,733 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:01,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:01,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:02,029 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:02,031 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:02,032 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:02,033 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:02,735 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:03,736 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:04,931 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:04,933 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:04,934 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:04,935 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:05,737 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:06,935 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:07,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:07,825 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:07,827 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:07,828 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:07,829 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:08,739 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:09,740 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:10,765 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:10,768 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:10,768 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:10,769 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:11,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:30:11,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:30:11,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:30:11,742 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:11,742 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:12,240 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:13,660 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:13,662 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:13,662 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:13,663 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:13,744 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:15,745 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:16,553 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:16,556 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:16,556 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:16,557 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:16,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:17,557 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:17,747 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:19,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:19,455 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:19,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:19,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:19,748 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:21,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:22,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:22,356 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:22,356 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:22,357 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:22,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:23,358 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:23,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:25,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:25,256 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:25,256 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:25,257 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:25,753 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:26,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:30:26,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:30:26,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:30:26,314 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:30:27,754 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:28,158 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:28,160 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:28,160 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:28,162 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:28,755 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:29,162 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:29,756 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:31,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:31,055 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:31,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:31,056 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:31,758 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:33,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:33,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:33,948 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:33,949 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:33,950 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:34,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:34,950 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:35,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:36,840 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:36,842 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:36,843 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:36,844 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:37,763 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:39,734 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:39,737 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:39,737 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:39,738 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:39,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:39,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:40,739 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:41,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:30:41,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:30:41,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:30:41,766 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:42,628 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:42,656 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:42,657 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:42,658 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:42,767 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:43,768 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:45,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:45,617 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:45,617 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:45,618 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:45,770 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:46,619 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:47,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:48,508 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:48,510 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:48,510 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:48,511 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:48,772 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:49,772 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:51,410 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:51,413 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:51,413 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:51,415 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:51,774 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:52,415 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:53,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:54,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:54,313 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:54,314 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:54,315 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:54,776 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:55,777 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:30:56,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:30:56,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:30:56,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:30:56,228 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:30:57,214 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:30:57,216 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:30:57,217 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:30:57,218 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:30:57,778 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:30:58,218 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:30:59,780 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:00,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:00,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:00,120 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:00,121 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:00,781 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:01,781 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:03,025 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:03,027 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:03,027 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:03,029 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:03,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:04,029 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:05,784 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:05,935 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:05,937 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:05,937 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:05,938 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:06,785 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:07,786 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:08,848 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:08,850 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:08,851 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:08,852 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:09,788 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:09,788 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:09,853 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:11,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:31:11,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:31:11,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:31:11,758 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:11,760 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:11,760 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:11,761 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:11,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:13,790 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:14,668 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:14,670 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:14,670 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:14,672 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:14,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:15,672 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:15,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:17,564 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:17,566 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:17,566 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:17,568 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:17,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:19,795 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:20,457 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:20,459 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:20,460 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:20,461 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:20,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:21,462 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:21,797 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:23,374 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:23,376 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:23,377 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:23,379 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:23,798 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:25,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:26,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:31:26,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:31:26,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:31:26,304 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:31:26,305 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:26,307 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:26,307 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:26,308 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:26,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:27,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:27,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:29,199 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:29,201 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:29,202 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:29,203 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:29,803 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:31,804 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:32,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:32,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:32,114 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:32,115 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:32,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:33,116 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:33,806 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:35,027 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:35,029 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:35,030 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:35,031 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:35,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:37,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:38,025 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:38,027 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:38,028 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:38,029 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:38,810 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:39,030 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:39,811 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:40,933 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:40,936 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:40,936 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:40,937 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:41,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:31:41,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:31:41,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:31:41,812 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:43,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:43,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:43,844 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:43,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:43,845 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:44,814 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:44,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:45,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:46,751 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:46,753 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:46,753 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:46,755 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:46,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:47,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:49,659 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:49,661 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:49,661 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:49,662 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:49,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:50,663 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:51,819 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:52,568 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:52,570 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:52,570 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:52,571 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:52,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:53,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:55,500 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:55,502 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:55,503 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:55,504 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:55,822 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:56,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:31:56,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:31:56,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:31:56,244 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:31:56,245 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:31:57,823 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:31:58,417 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:31:58,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:31:58,419 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:31:58,420 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:31:58,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:31:59,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:01,335 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:01,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:01,338 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:01,338 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:01,339 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:01,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:03,827 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:04,252 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:04,254 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:04,255 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:04,256 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:04,828 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:05,828 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:07,164 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:07,167 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:07,167 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:07,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:07,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:07,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:09,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:10,095 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:10,097 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:10,098 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:10,099 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:10,831 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:11,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:32:11,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:32:11,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:32:11,832 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:12,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:13,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:13,015 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:13,015 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:13,016 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:13,833 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:15,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:15,953 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:15,955 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:15,956 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:15,957 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:16,835 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:17,836 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:17,957 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:18,879 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:18,881 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:18,882 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:18,883 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:19,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:19,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:21,792 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:21,795 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:21,796 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:21,797 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:21,839 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:23,797 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:23,840 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:24,776 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:24,778 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:24,779 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:24,780 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:24,841 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:25,841 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:26,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:32:26,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:32:26,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:32:26,272 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:32:27,695 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:27,697 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:27,697 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:27,698 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:27,843 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:29,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:29,844 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:30,611 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:30,613 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:30,614 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:30,615 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:30,845 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:31,845 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:33,531 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:33,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:33,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:33,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:33,847 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:35,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:35,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:36,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:36,455 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:36,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:36,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:36,849 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:37,849 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:39,374 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:39,377 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:39,377 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:39,378 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:39,851 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:41,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:32:41,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:32:41,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:32:41,261 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:41,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:42,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:42,296 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:42,297 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:42,298 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:42,853 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:43,853 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:45,231 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:45,234 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:45,234 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:45,235 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:45,855 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:46,191 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:46,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:47,856 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:49,357 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:49,360 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:49,360 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:49,361 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:49,857 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:51,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:51,858 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:52,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:52,288 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:52,289 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:52,290 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:52,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:53,860 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:55,218 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:55,220 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:55,220 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:55,221 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:55,861 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:56,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:32:56,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:32:56,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:32:56,258 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:32:57,259 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:32:57,862 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:32:58,139 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:32:58,142 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:32:58,142 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:32:58,143 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:32:58,863 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:32:59,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:01,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:01,071 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:01,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:01,073 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:01,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:03,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:03,867 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:03,995 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:03,998 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:03,998 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:03,999 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:04,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:05,869 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:06,969 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:06,974 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:06,974 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:06,976 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:07,871 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:07,871 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:08,976 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:09,887 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:09,890 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:09,890 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:09,891 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:10,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:11,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:33:11,060 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:33:11,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:33:11,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:12,809 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:12,811 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:12,811 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:12,812 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:12,875 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:13,875 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:14,813 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:15,775 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:15,777 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:15,777 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:15,779 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:15,877 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:17,878 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:18,715 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:18,717 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:18,717 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:18,718 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:18,879 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:19,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:20,719 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:21,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:21,639 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:21,639 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:21,640 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:21,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:23,883 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:24,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:24,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:24,557 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:24,558 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:24,884 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:25,884 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:26,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:33:26,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:33:26,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:33:26,331 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:33:26,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:27,472 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:27,474 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:27,475 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:27,476 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:27,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:29,887 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:30,418 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:30,420 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:30,421 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:30,422 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:30,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:31,422 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:31,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:33,333 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:33,336 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:33,336 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:33,337 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:33,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:35,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:36,250 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:36,252 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:36,253 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:36,254 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:36,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:37,254 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:37,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:39,187 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:39,190 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:39,190 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:39,192 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:39,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:41,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:33:41,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:33:41,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:33:41,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:42,102 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:42,105 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:42,105 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:42,106 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:42,899 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:43,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:43,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:45,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:45,018 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:45,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:45,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:45,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:45,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:47,952 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:47,955 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:47,955 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:47,956 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:48,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:48,957 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:49,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:50,956 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:50,959 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:50,959 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:50,960 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:51,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:51,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:53,871 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:53,873 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:53,874 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:53,875 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:53,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:54,875 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:33:55,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:56,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:33:56,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:33:56,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:33:56,334 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:33:56,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:56,790 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:56,790 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:56,791 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:56,910 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:33:57,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:33:59,700 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:33:59,702 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:33:59,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:33:59,705 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:33:59,912 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:00,705 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:01,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:02,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:02,616 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:02,617 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:02,618 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:02,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:03,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:05,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:05,545 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:05,546 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:05,547 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:05,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:06,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:07,918 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:08,456 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:08,458 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:08,459 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:08,460 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:08,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:09,920 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:11,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:34:11,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:34:11,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:34:11,380 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:11,382 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:11,382 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:11,383 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:11,922 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:12,384 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:13,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:14,292 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:14,294 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:14,295 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:14,296 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:14,924 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:15,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:17,207 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:17,209 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:17,210 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:17,211 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:17,926 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:18,211 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:19,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:20,122 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:20,124 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:20,124 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:20,126 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:20,928 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:21,929 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:23,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:23,044 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:23,044 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:23,045 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:23,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:23,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:24,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:25,972 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:25,974 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:25,975 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:25,976 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:26,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:34:26,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:34:26,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:34:26,321 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:34:26,933 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:27,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:28,888 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:28,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:28,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:28,893 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:28,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:29,893 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:29,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:31,822 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:31,825 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:31,825 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:31,827 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:31,937 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:33,938 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:34,824 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:34,827 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:34,827 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:34,828 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:34,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:35,829 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:35,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:37,738 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:37,740 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:37,740 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:37,742 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:37,942 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:39,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:40,652 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:40,654 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:40,654 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:40,655 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:40,944 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:41,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:34:41,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:34:41,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:34:41,339 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:41,945 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:43,580 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:43,582 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:43,583 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:43,584 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:43,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:45,948 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:46,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:46,520 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:46,520 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:46,520 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:46,522 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:46,949 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:47,949 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:49,431 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:49,434 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:49,434 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:49,435 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:49,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:51,952 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:52,346 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:52,350 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:52,351 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:52,351 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:52,352 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:52,953 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:53,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:55,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:55,262 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:55,263 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:55,264 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:55,956 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:56,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:34:56,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:34:56,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:34:56,305 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:34:57,957 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:34:58,175 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:34:58,178 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:34:58,178 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:34:58,178 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:34:58,179 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:34:58,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:34:59,959 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:01,111 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:01,113 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:01,114 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:01,115 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:01,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:01,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:04,026 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:04,028 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:04,029 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:04,030 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:04,031 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:04,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:05,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:06,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:06,939 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:06,940 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:06,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:06,964 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:07,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:09,850 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:09,852 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:09,853 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:09,853 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:09,854 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:09,967 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:11,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:35:11,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:35:11,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:35:11,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:12,800 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:12,802 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:12,803 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:12,804 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:12,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:13,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:15,805 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:15,817 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:15,820 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:15,820 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:15,821 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:15,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:17,973 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:18,736 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:18,738 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:18,738 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:18,739 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:18,974 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:19,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:21,656 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:21,659 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:21,660 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:21,660 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:21,661 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:21,977 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:23,978 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:24,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:24,576 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:24,576 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:24,578 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:24,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:25,980 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:26,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:35:26,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:35:26,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:35:26,309 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:35:27,309 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:27,498 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:27,500 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:27,500 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:27,501 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:27,981 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:29,982 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:30,443 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:30,445 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:30,446 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:30,447 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:30,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:31,984 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:32,448 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:33,389 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:33,391 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:33,391 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:33,392 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:33,985 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:35,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:36,314 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:36,316 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:36,316 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:36,317 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:36,987 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:37,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:38,318 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:39,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:39,291 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:39,291 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:39,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:39,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:41,061 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:35:41,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:35:41,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:35:41,991 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:42,262 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:42,264 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:42,265 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:42,266 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:42,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:43,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:44,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:45,186 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:45,188 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:45,189 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:45,190 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:45,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:45,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:48,110 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:48,113 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:48,113 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:48,114 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:48,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:49,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:50,115 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:51,031 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:51,033 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:51,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:51,035 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:51,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:51,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:53,948 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:53,950 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:53,951 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:53,952 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:53,999 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:55,952 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:35:56,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:56,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:35:56,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:35:56,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:35:56,251 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:35:56,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:56,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:56,871 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:56,872 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:35:57,001 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:35:58,002 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:35:59,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:35:59,791 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:35:59,791 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:35:59,793 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:00,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:01,793 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:02,005 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:02,798 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:02,801 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:02,801 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:02,802 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:03,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:04,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:05,719 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:05,721 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:05,721 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:05,723 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:06,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:07,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:08,009 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:08,645 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:08,647 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:08,648 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:08,649 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:09,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:10,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:11,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:36:11,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:36:11,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:36:11,567 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:11,571 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:11,572 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:11,573 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:12,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:13,573 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:14,013 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:14,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:14,490 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:14,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:14,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:15,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:16,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:17,414 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:17,416 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:17,417 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:17,418 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:18,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:19,418 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:20,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:20,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:20,343 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:20,344 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:20,345 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:21,018 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:22,018 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:23,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:23,293 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:23,294 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:23,295 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:24,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:25,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:26,021 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:26,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:36:26,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:36:26,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:36:26,194 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:26,299 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:36:26,299 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:26,299 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:26,300 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:27,022 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:28,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:29,118 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:29,121 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:29,121 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:29,122 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:30,024 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:30,024 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:31,123 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:32,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:32,045 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:32,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:32,047 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:33,026 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:34,026 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:34,968 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:34,970 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:34,971 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:34,972 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:35,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:36,028 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:36,972 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:37,892 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:37,895 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:37,895 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:37,900 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:38,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:40,030 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:40,817 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:40,819 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:40,819 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:40,821 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:41,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:41,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:36:41,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:36:41,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:36:42,032 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:42,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:43,832 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:43,835 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:43,835 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:43,836 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:44,034 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:46,035 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:46,773 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:46,775 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:46,775 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:46,777 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:47,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:47,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:48,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:49,691 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:49,694 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:49,694 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:49,695 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:50,038 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:52,039 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:52,607 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:52,610 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:52,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:52,612 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:53,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:53,612 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:36:54,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:55,524 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:55,526 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:55,527 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:55,528 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:56,042 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:56,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:36:56,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:36:56,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:36:56,229 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:36:58,043 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:36:58,439 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:36:58,442 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:36:58,442 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:36:58,443 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:36:59,044 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:36:59,444 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:00,044 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:01,357 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:01,359 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:01,360 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:01,361 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:02,046 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:04,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:04,281 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:04,283 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:04,284 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:04,285 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:05,048 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:05,285 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:06,048 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:07,197 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:07,199 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:07,200 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:07,201 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:08,049 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:08,050 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:10,115 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:10,117 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:10,118 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:10,119 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:11,051 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:11,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:37:11,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:37:11,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:37:11,251 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:12,052 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:13,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:13,037 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:13,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:13,038 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:13,053 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:14,053 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:15,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:15,956 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:15,957 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:15,958 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:16,055 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:16,958 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:18,055 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:18,872 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:18,875 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:18,876 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:18,877 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:19,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:20,057 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:21,790 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:21,792 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:21,793 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:21,794 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:22,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:22,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:24,059 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:24,706 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:24,709 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:24,709 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:24,710 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:25,061 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:26,061 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:26,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:37:26,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:37:26,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:37:26,342 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:37:27,668 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:27,671 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:27,671 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:27,672 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:28,063 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:28,673 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:30,064 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:30,586 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:30,589 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:30,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:30,590 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:31,065 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:32,066 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:33,532 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:33,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:33,535 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:33,536 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:34,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:34,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:36,069 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:36,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:36,456 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:36,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:36,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:37,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:38,071 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:39,388 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:39,390 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:39,391 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:39,392 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:40,073 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:40,392 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:41,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:37:41,062 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:37:41,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:37:42,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:42,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:42,313 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:42,313 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:42,315 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:43,075 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:44,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:45,235 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:45,236 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:45,237 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:45,238 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:46,077 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:46,077 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:46,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:48,159 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:48,161 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:48,162 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:48,163 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:49,079 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:50,080 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:51,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:51,082 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:51,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:51,084 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:52,084 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:52,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:52,085 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:54,013 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:54,016 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:54,016 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:54,018 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:54,086 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:56,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:37:56,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:37:56,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:37:56,087 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:56,300 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:37:56,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:56,940 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:56,940 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:56,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:37:57,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:37:57,942 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:37:58,089 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:37:59,881 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:37:59,884 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:37:59,884 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:37:59,886 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:00,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:02,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:02,805 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:02,808 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:02,808 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:02,809 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:03,093 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:03,810 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:04,094 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:05,730 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:05,732 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:05,732 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:05,734 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:06,095 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:08,096 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:08,652 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:08,655 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:08,656 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:08,657 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:09,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:09,657 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:10,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:11,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:38:11,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:38:11,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:38:11,665 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:11,667 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:11,668 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:11,669 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:12,100 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:14,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:14,584 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:14,586 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:14,587 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:14,588 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:15,102 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:15,589 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:16,103 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:17,525 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:17,527 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:17,527 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:17,529 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:18,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:20,106 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:20,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:20,469 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:20,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:20,470 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:21,107 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:21,471 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:22,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:23,392 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:23,395 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:23,395 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:23,396 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:24,110 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:26,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:38:26,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:38:26,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:38:26,111 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:26,301 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:38:26,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:26,321 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:26,322 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:26,323 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:27,112 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:27,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:28,113 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:29,246 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:29,248 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:29,248 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:29,250 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:30,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:30,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:32,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:32,165 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:32,166 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:32,167 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:33,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:33,168 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:34,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:35,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:35,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:35,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:35,086 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:35,118 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:36,119 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:38,017 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:38,019 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:38,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:38,021 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:38,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:39,021 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:40,122 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:40,934 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:40,936 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:40,936 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:40,937 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:41,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:38:41,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:38:41,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:38:41,123 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:42,124 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:43,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:43,854 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:43,855 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:43,856 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:44,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:44,857 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:46,127 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:46,770 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:46,772 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:46,773 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:46,774 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:47,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:48,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:49,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:49,696 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:49,696 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:49,697 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:50,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:50,698 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:52,131 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:52,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:52,620 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:52,621 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:52,622 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:53,132 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:54,133 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:55,636 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:55,638 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:55,638 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:55,639 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:56,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:38:56,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:38:56,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:38:56,135 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:38:56,334 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:38:56,334 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:38:58,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:38:58,576 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:38:58,582 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:38:58,582 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:38:58,583 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:38:59,137 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:00,138 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:01,499 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:01,502 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:01,502 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:01,503 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:01,504 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:02,140 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:04,141 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:04,422 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:04,424 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:04,424 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:04,426 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:05,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:06,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:07,345 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:07,347 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:07,348 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:07,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:07,349 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:08,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:08,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:10,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:10,268 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:10,268 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:10,269 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:11,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:39:11,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:39:11,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:39:11,147 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:12,148 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:13,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:13,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:13,192 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:13,192 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:13,193 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:14,149 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:14,149 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:16,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:16,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:16,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:16,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:16,151 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:18,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:19,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:19,037 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:19,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:19,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:19,039 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:19,153 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:20,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:21,971 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:21,973 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:21,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:21,975 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:22,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:24,157 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:24,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:24,893 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:24,894 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:24,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:24,895 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:25,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:26,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:39:26,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:39:26,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:39:26,159 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:26,335 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:39:27,813 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:27,816 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:27,817 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:27,818 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:28,160 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:30,162 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:30,733 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:30,736 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:30,736 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:30,736 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:30,737 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:31,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:32,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:33,652 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:33,655 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:33,655 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:33,656 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:34,165 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:36,166 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:36,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:36,611 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:36,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:36,611 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:36,612 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:37,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:38,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:39,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:39,620 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:39,620 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:39,621 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:40,170 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:41,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:39:41,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:39:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:39:42,171 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:42,328 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:42,556 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:42,558 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:42,559 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:42,560 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:43,172 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:44,172 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:45,515 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:45,517 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:45,518 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:45,519 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:46,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:47,519 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:48,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:48,434 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:48,436 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:48,437 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:48,438 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:49,176 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:50,176 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:51,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:51,355 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:51,356 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:51,357 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:52,178 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:52,178 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:53,357 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:39:54,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:54,302 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:54,302 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:54,303 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:55,180 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:56,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:39:56,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:39:56,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:39:56,181 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:56,233 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:39:57,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:39:57,234 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:39:57,234 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:39:57,235 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:39:58,182 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:39:58,183 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:39:59,236 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:00,159 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:00,162 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:00,162 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:00,163 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:00,184 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:02,185 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:03,096 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:03,098 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:03,098 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:03,100 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:03,186 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:04,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:05,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:06,032 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:06,034 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:06,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:06,036 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:06,188 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:08,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:08,960 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:08,962 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:08,963 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:08,964 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:09,190 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:10,190 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:10,964 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:11,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:40:11,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:40:11,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:40:11,885 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:11,889 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:11,890 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:11,891 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:12,192 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:14,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:14,831 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:14,833 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:14,833 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:14,835 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:15,194 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:16,194 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:16,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:17,758 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:17,761 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:17,761 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:17,762 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:18,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:20,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:20,685 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:20,687 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:20,688 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:20,689 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:21,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:22,198 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:22,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:23,702 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:23,704 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:23,705 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:23,706 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:24,199 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:26,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:40:26,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:40:26,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:40:26,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:26,330 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:40:26,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:26,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:26,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:26,633 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:27,201 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:28,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:28,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:29,546 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:29,549 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:29,549 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:29,550 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:30,203 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:32,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:32,489 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:32,491 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:32,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:32,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:33,205 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:34,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:34,493 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:35,451 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:35,453 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:35,454 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:35,455 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:36,207 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:36,207 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:38,398 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:38,401 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:38,401 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:38,402 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:39,209 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:40,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:40,403 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:41,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:40:41,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:40:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:40:41,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:41,326 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:41,327 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:41,328 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:42,211 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:42,211 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:44,269 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:44,271 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:44,271 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:44,272 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:45,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:46,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:46,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:47,202 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:47,204 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:47,205 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:47,206 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:47,214 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:48,215 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:50,146 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:50,148 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:50,149 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:50,151 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:50,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:52,151 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:52,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:53,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:53,071 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:53,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:53,073 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:53,218 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:54,219 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:55,992 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:55,995 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:55,995 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:55,996 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:56,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:40:56,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:40:56,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:40:56,220 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:40:56,297 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:40:57,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:40:58,221 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:40:58,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:40:58,922 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:40:58,922 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:40:58,923 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:40:59,222 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:00,223 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:01,864 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:01,866 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:01,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:01,868 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:02,224 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:02,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:04,225 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:04,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:04,802 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:04,802 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:04,803 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:05,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:06,227 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:07,863 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:07,865 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:07,866 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:07,867 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:08,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:08,867 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:10,229 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:10,780 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:10,783 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:10,783 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:10,784 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:11,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:41:11,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:41:11,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:41:11,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:12,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:13,716 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:13,719 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:13,719 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:13,720 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:14,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:14,721 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:16,233 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:16,651 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:16,653 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:16,654 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:16,655 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:17,234 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:18,234 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:19,577 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:19,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:19,580 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:19,581 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:20,236 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:20,236 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:20,581 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:22,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:22,506 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:22,506 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:22,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:23,238 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:24,238 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:25,432 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:25,434 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:25,435 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:25,436 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:26,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:41:26,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:41:26,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:41:26,240 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:26,240 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:26,310 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:41:26,310 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:28,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:28,366 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:28,366 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:28,367 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:29,242 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:30,242 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:31,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:31,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:31,292 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:31,294 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:32,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:32,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:32,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:34,238 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:34,240 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:34,240 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:34,241 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:34,245 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:36,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:37,164 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:37,166 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:37,166 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:37,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:37,247 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:38,168 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:38,248 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:40,089 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:40,091 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:40,092 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:40,093 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:40,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:41,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:41:41,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:41:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:41:42,250 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:43,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:43,037 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:43,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:43,039 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:43,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:44,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:44,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:45,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:45,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:45,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:45,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:46,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:48,254 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:48,898 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:48,900 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:48,901 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:48,902 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:49,255 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:49,902 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:50,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:51,907 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:51,910 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:51,910 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:51,911 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:52,257 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:54,258 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:54,829 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:54,831 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:54,832 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:54,833 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:55,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:41:55,834 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:41:56,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:41:56,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:41:56,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:41:56,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:41:56,326 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:41:57,750 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:41:57,752 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:41:57,753 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:41:57,754 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:41:58,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:00,262 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:00,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:00,674 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:00,674 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:00,675 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:01,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:01,676 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:02,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:03,593 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:03,595 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:03,596 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:03,597 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:04,265 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:04,265 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:06,537 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:06,540 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:06,540 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:06,541 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:07,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:07,542 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:08,268 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:09,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:09,468 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:09,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:09,470 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:10,269 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:10,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:11,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:42:11,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:42:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:42:12,389 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:12,392 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:12,392 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:12,393 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:13,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:13,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:14,272 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:15,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:15,329 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:15,330 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:15,331 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:16,273 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:16,273 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:18,272 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:18,298 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:18,300 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:18,301 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:19,301 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:19,301 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:20,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:21,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:21,223 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:21,223 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:21,224 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:21,303 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:22,303 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:24,146 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:24,149 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:24,149 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:24,151 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:24,304 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:25,151 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:26,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:42:26,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:42:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:42:26,231 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:42:26,305 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:27,105 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:27,109 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:27,109 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:27,111 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:27,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:28,307 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:30,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:30,037 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:30,037 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:30,038 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:30,308 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:31,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:32,309 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:33,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:33,080 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:33,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:33,081 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:33,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:33,762 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:34,311 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:36,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:36,895 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:36,897 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:36,898 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:36,898 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:36,899 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:37,313 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:38,313 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:39,834 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:39,836 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:39,837 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:39,838 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:40,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:41,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:42:41,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:42:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:42:42,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:42,315 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:42,766 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:42,768 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:42,768 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:42,770 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:43,316 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:44,317 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:45,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:45,696 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:45,696 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:45,697 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:46,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:46,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:47,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:48,644 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:48,646 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:48,646 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:48,648 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:49,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:50,321 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:51,594 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:51,595 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:51,596 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:51,597 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:52,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:52,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:53,598 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:42:54,521 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:54,523 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:54,524 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:54,526 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:55,324 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:56,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:42:56,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:42:56,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:42:56,325 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:56,334 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:42:57,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:42:57,454 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:42:57,455 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:42:57,456 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:42:58,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:42:58,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:42:59,457 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:00,382 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:00,384 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:00,384 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:00,385 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:01,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:02,329 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:03,322 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:03,324 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:03,325 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:03,326 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:03,330 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:04,330 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:05,326 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:06,251 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:06,255 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:06,255 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:06,256 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:06,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:08,333 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:09,179 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:09,182 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:09,182 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:09,183 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:09,334 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:10,335 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:11,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:43:11,064 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:43:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:43:11,258 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:12,148 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:12,150 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:12,151 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:12,152 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:12,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:14,337 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:15,099 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:15,102 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:15,102 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:15,103 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:15,338 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:16,339 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:17,104 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:18,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:18,076 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:18,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:18,078 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:18,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:20,341 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:20,992 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:20,995 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:20,995 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:20,996 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:21,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:22,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:22,997 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:23,932 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:23,934 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:23,935 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:23,936 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:24,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:26,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:43:26,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:43:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:43:26,322 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:43:26,345 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:26,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:26,854 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:26,854 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:26,855 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:27,346 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:28,346 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:28,856 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:29,809 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:29,812 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:29,812 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:29,813 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:30,348 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:32,349 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:32,748 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:32,750 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:32,751 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:32,752 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:33,350 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:34,350 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:34,753 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:35,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:35,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:35,673 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:35,675 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:36,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:36,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:38,615 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:38,617 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:38,617 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:38,619 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:39,353 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:40,354 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:40,619 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:41,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:43:41,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:43:41,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:43:41,551 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:41,553 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:41,553 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:41,555 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:42,355 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:42,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:44,479 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:44,482 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:44,482 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:44,483 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:45,357 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:46,358 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:46,484 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:47,412 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:47,414 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:47,415 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:47,418 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:48,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:48,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:50,358 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:50,359 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:50,360 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:50,362 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:51,362 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:52,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:52,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:53,313 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:53,315 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:53,316 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:53,317 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:53,364 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:54,364 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:56,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:43:56,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:43:56,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:43:56,259 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:43:56,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:56,264 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:56,265 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:56,266 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:56,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:43:58,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:43:58,366 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:43:59,199 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:43:59,201 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:43:59,201 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:43:59,203 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:43:59,367 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:00,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:02,172 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:02,174 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:02,175 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:02,176 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:02,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:04,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:04,370 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:05,101 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:05,103 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:05,104 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:05,105 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:05,371 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:06,372 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:08,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:08,068 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:08,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:08,069 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:08,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:10,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:10,374 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:10,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:10,998 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:10,999 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:11,000 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:11,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:44:11,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:44:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:44:11,375 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:12,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:13,923 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:13,926 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:13,926 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:13,927 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:14,377 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:15,928 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:16,378 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:16,854 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:16,857 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:16,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:16,858 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:17,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:18,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:19,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:19,787 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:19,787 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:19,788 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:20,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:20,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:21,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:22,732 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:22,734 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:22,735 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:22,736 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:23,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:24,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:25,670 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:25,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:25,673 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:25,674 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:26,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:44:26,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:44:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:44:26,340 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:44:26,385 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:26,385 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:27,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:28,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:28,610 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:28,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:28,612 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:29,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:30,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:31,577 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:31,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:31,580 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:31,581 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:32,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:32,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:32,581 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:34,565 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:34,567 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:34,568 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:34,569 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:35,391 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:36,391 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:37,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:37,506 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:37,507 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:37,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:38,393 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:38,393 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:38,508 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:40,446 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:40,448 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:40,449 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:40,450 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:44:41,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:44:41,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:44:41,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:42,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:43,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:43,397 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:43,397 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:43,398 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:44,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:44,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:44,399 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:46,443 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:46,445 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:46,446 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:46,447 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:47,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:48,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:49,377 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:49,379 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:49,380 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:49,381 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:49,402 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:50,381 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:50,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:52,302 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:52,305 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:52,305 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:52,306 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:52,404 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:54,405 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:55,231 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:55,233 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:55,234 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:55,235 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:55,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:44:56,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:44:56,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:44:56,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:44:56,319 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:44:56,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:44:56,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:44:58,175 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:44:58,177 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:44:58,178 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:44:58,179 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:44:58,408 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:00,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:01,126 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:01,129 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:01,129 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:01,130 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:01,410 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:02,131 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:02,410 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:04,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:04,068 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:04,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:04,070 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:04,412 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:06,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:06,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:06,998 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:06,999 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:07,000 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:07,414 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:08,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:08,414 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:09,922 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:09,924 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:09,924 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:09,925 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:10,416 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:11,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:45:11,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:45:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:45:12,417 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:12,847 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:12,851 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:12,852 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:12,853 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:13,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:13,854 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:14,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:15,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:15,779 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:15,780 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:15,781 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:16,420 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:16,420 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:18,744 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:18,746 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:18,747 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:18,748 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:19,422 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:19,748 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:20,422 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:21,672 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:21,674 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:21,675 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:21,676 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:22,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:22,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:24,598 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:24,600 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:24,601 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:24,602 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:25,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:25,602 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:26,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:45:26,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:45:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:45:26,252 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:45:26,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:27,540 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:27,542 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:27,542 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:27,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:28,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:28,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:30,568 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:30,570 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:30,571 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:30,572 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:31,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:31,572 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:32,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:33,494 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:33,497 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:33,497 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:33,498 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:34,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:34,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:36,468 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:36,471 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:36,471 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:36,472 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:37,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:37,473 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:38,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:39,394 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:39,396 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:39,397 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:39,398 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:39,435 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:40,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:45:41,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:45:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:45:42,336 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:42,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:42,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:42,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:42,437 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:43,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:44,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:45,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:45,265 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:45,266 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:45,267 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:45,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:46,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:48,200 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:48,203 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:48,203 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:48,204 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:48,441 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:49,205 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:50,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:51,126 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:51,128 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:51,129 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:51,130 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:51,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:52,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:54,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:54,054 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:54,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:54,055 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:54,445 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:55,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:45:56,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:45:56,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:45:56,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:45:56,323 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:45:56,446 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:56,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:56,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:56,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:56,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:45:57,447 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:45:58,447 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:45:59,913 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:45:59,915 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:45:59,915 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:45:59,917 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:00,449 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:00,449 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:00,917 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:02,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:02,858 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:02,858 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:02,859 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:03,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:04,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:05,802 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:05,804 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:05,804 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:05,805 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:06,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:06,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:06,806 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:08,730 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:08,732 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:08,733 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:08,734 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:09,455 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:10,455 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:11,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:46:11,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:46:11,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:46:11,667 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:11,669 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:11,670 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:11,671 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:12,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:12,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:12,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:14,704 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:14,706 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:14,707 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:14,708 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:15,459 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:16,459 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:17,632 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:17,635 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:17,635 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:17,636 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:18,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:18,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:18,637 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:20,560 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:20,563 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:20,563 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:20,564 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:21,463 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:22,464 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:23,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:23,491 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:23,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:23,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:24,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:24,465 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:24,493 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:26,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:46:26,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:46:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:46:26,260 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:46:26,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:26,422 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:26,423 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:26,424 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:26,467 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:28,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:29,351 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:29,353 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:29,353 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:29,354 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:29,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:30,355 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:30,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:32,281 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:32,283 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:32,284 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:32,285 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:32,471 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:34,471 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:35,214 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:35,216 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:35,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:35,217 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:35,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:36,218 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:36,473 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:38,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:38,166 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:38,167 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:38,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:38,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:40,475 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:41,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:46:41,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:46:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:46:41,113 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:41,320 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:41,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:41,320 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:41,321 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:41,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:42,477 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:44,033 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:44,035 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:44,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:44,037 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:44,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:46,479 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:46,957 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:46,959 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:46,959 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:46,960 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:46,961 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:47,480 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:48,481 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:49,885 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:49,887 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:49,887 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:49,888 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:50,482 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:50,482 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:52,811 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:52,814 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:52,814 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:52,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:52,815 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:53,484 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:54,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:55,831 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:55,834 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:55,834 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:55,835 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:56,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:46:56,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:46:56,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:46:56,262 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:46:56,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:46:56,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:46:58,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:46:58,767 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:46:58,769 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:46:58,770 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:46:58,772 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:46:59,488 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:00,489 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:01,694 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:01,696 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:01,696 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:01,697 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:02,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:02,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:03,698 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:04,628 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:04,630 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:04,630 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:04,632 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:05,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:06,493 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:07,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:07,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:07,557 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:07,558 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:08,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:08,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:09,558 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:10,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:10,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:10,484 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:10,485 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:10,495 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:47:11,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:47:11,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:47:12,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:13,411 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:13,413 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:13,414 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:13,415 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:13,497 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:14,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:15,416 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:16,337 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:16,339 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:16,340 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:16,341 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:16,500 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:18,501 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:19,279 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:19,284 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:19,284 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:19,285 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:19,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:20,503 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:21,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:22,204 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:22,206 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:22,207 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:22,208 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:22,504 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:24,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:25,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:25,134 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:25,134 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:25,135 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:25,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:47:26,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:47:26,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:47:26,278 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:47:26,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:27,278 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:28,101 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:28,105 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:28,107 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:28,108 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:28,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:30,510 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:31,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:31,044 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:31,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:31,046 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:31,511 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:32,512 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:33,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:33,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:33,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:33,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:33,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:34,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:34,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:36,910 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:36,912 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:36,913 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:36,914 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:37,516 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:38,517 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:38,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:39,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:39,931 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:39,931 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:39,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:40,518 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:40,518 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:41,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:47:41,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:47:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:47:42,869 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:42,871 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:42,871 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:42,872 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:43,521 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:44,521 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:44,873 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:45,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:45,801 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:45,801 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:45,802 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:46,523 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:46,523 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:48,765 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:48,767 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:48,768 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:48,770 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:49,525 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:50,526 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:50,771 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:51,694 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:51,697 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:51,697 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:51,698 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:52,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:52,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:54,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:54,620 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:54,620 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:54,621 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:55,530 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:47:56,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:47:56,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:47:56,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:47:56,331 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:47:56,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:47:56,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:57,561 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:47:57,563 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:47:57,564 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:47:57,565 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:47:58,532 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:47:58,532 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:00,494 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:00,520 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:00,521 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:00,522 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:00,534 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:01,523 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:02,535 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:03,416 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:03,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:03,419 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:03,420 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:03,536 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:04,537 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:06,344 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:06,346 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:06,347 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:06,348 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:06,539 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:07,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:08,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:09,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:09,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:09,292 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:09,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:09,541 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:10,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:48:11,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:48:11,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:48:12,231 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:12,233 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:12,233 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:12,234 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:12,543 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:13,235 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:14,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:15,166 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:15,168 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:15,169 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:15,170 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:15,545 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:16,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:18,111 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:18,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:18,114 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:18,115 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:18,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:18,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:19,115 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:21,173 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:21,175 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:21,176 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:21,177 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:21,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:22,551 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:24,098 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:24,101 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:24,101 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:24,102 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:24,552 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:24,553 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:25,102 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:48:26,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:48:26,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:48:26,276 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:48:27,021 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:27,023 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:27,023 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:27,025 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:27,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:28,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:29,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:29,972 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:29,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:29,974 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:30,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:30,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:30,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:32,905 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:32,908 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:32,908 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:32,910 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:33,559 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:34,560 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:35,850 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:35,852 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:35,852 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:35,853 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:36,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:36,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:36,854 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:38,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:38,801 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:38,801 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:38,802 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:39,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:40,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:41,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:48:41,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:48:41,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:48:41,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:41,727 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:41,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:41,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:42,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:42,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:42,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:44,674 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:44,676 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:44,677 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:44,678 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:45,568 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:46,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:47,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:47,617 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:47,617 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:47,618 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:48,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:48,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:48,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:50,556 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:50,558 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:50,559 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:50,560 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:50,573 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:52,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:53,487 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:53,489 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:53,490 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:53,491 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:53,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:54,491 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:48:54,576 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:56,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:48:56,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:48:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:48:56,337 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:48:56,416 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:56,418 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:56,418 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:56,419 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:56,577 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:48:58,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:48:59,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:48:59,365 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:48:59,365 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:48:59,366 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:48:59,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:00,367 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:00,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:02,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:02,307 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:02,307 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:02,308 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:02,582 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:04,583 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:05,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:05,330 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:05,330 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:05,331 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:05,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:06,332 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:06,585 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:08,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:08,255 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:08,255 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:08,256 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:08,587 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:10,588 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:11,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:49:11,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:49:11,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:49:11,182 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:11,285 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:11,285 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:11,286 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:11,589 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:12,287 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:12,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:14,116 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:14,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:14,119 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:14,120 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:14,591 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:14,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:17,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:17,050 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:17,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:17,051 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:17,594 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:18,052 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:18,594 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:19,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:19,976 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:19,976 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:19,978 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:20,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:20,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:22,899 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:22,902 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:22,902 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:22,903 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:23,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:23,904 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:24,599 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:25,833 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:25,835 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:25,835 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:25,836 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:26,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:49:26,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:49:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:49:26,292 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:49:26,601 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:26,601 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:28,778 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:28,780 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:28,781 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:28,782 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:29,603 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:29,782 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:30,604 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:31,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:31,727 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:31,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:31,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:32,605 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:32,606 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:34,674 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:34,677 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:34,677 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:34,679 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:35,607 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:35,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:36,608 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:37,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:37,623 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:37,624 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:37,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:38,609 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:38,610 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:40,546 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:40,548 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:40,549 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:40,550 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:40,611 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:49:41,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:49:41,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:49:41,245 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:42,612 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:43,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:43,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:43,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:43,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:43,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:44,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:46,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:46,516 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:46,518 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:46,519 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:46,520 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:46,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:48,616 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:49,446 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:49,448 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:49,449 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:49,450 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:49,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:50,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:52,390 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:52,393 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:52,393 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:52,393 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:52,394 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:52,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:54,620 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:55,318 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:55,320 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:55,321 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:55,322 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:55,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:49:56,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:49:56,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:49:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:49:56,295 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:49:56,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:49:58,248 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:49:58,250 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:49:58,251 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:49:58,251 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:49:58,252 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:49:58,623 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:00,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:01,170 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:01,173 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:01,173 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:01,174 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:01,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:02,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:04,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:04,093 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:04,093 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:04,094 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:04,095 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:04,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:04,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:07,017 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:07,020 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:07,020 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:07,022 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:07,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:08,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:09,938 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:09,940 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:09,941 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:09,941 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:09,942 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:10,631 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:10,631 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:11,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:50:11,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:50:11,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:50:12,858 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:12,861 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:12,861 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:12,862 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:13,633 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:14,633 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:15,809 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:15,812 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:15,812 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:15,812 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:15,813 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:16,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:17,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:18,735 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:18,738 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:18,738 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:18,739 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:19,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:21,637 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:21,665 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:21,667 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:21,668 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:21,668 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:21,669 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:22,639 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:23,639 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:24,594 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:24,596 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:24,596 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:24,597 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:24,640 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:26,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:50:26,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:50:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:50:26,292 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:50:27,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:27,617 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:27,620 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:27,620 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:27,621 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:27,642 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:27,642 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:29,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:30,545 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:30,547 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:30,548 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:30,549 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:30,644 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:32,549 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:33,465 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:33,467 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:33,468 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:33,469 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:33,646 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:33,646 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:35,647 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:36,419 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:36,422 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:36,422 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:36,423 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:36,648 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:38,424 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:39,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:39,354 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:39,354 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:39,355 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:39,650 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:39,650 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:50:41,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:50:41,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:50:41,651 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:42,303 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:42,305 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:42,305 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:42,307 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:42,652 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:44,307 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:45,230 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:45,233 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:45,233 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:45,234 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:45,654 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:45,654 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:47,655 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:48,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:48,155 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:48,155 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:48,156 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:48,655 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:49,656 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:50,157 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:51,087 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:51,089 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:51,089 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:51,090 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:51,657 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:53,658 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:54,012 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:54,014 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:54,015 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:54,016 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:54,659 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:55,660 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:56,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:50:56,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:50:56,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:50:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:50:56,265 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:50:56,937 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:56,939 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:56,939 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:56,941 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:50:57,661 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:50:59,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:50:59,886 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:50:59,888 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:50:59,889 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:50:59,890 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:00,663 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:01,664 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:01,890 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:02,844 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:02,847 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:02,847 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:02,848 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:03,665 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:05,666 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:05,797 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:05,800 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:05,800 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:05,801 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:06,667 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:07,668 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:07,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:08,734 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:08,736 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:08,737 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:08,741 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:09,669 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:11,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:51:11,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:51:11,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:51:11,670 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:11,800 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:11,802 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:11,803 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:11,804 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:12,671 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:12,804 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:13,672 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:14,732 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:14,734 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:14,734 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:14,735 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:15,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:17,674 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:17,676 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:17,676 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:17,677 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:17,677 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:18,678 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:18,678 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:19,678 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:20,603 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:20,605 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:20,605 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:20,606 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:20,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:23,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:23,545 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:23,546 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:23,547 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:23,682 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:23,682 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:24,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:25,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:26,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:51:26,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:51:26,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:51:26,314 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:51:26,492 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:26,494 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:26,494 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:26,495 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:26,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:29,431 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:29,433 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:29,433 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:29,435 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:29,686 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:29,686 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:30,435 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:31,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:32,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:32,367 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:32,368 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:32,369 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:32,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:35,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:35,297 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:35,298 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:35,300 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:35,689 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:35,690 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:36,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:37,690 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:38,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:38,241 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:38,242 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:38,243 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:38,691 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:39,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:51:41,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:51:41,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:51:41,171 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:41,256 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:41,256 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:41,258 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:41,693 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:42,259 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:43,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:44,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:44,115 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:44,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:44,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:44,695 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:45,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:47,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:47,046 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:47,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:47,047 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:47,697 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:48,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:49,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:49,989 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:49,991 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:49,992 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:49,993 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:50,699 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:51,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:53,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:53,043 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:53,043 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:53,044 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:53,701 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:54,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:51:55,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:55,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:55,979 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:55,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:55,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:56,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:51:56,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:51:56,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:51:56,340 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:51:56,704 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:57,704 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:51:58,930 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:51:58,932 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:51:58,932 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:51:58,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:51:59,706 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:51:59,934 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:01,707 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:01,872 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:01,875 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:01,875 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:01,877 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:02,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:03,709 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:04,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:04,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:04,837 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:04,838 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:05,711 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:05,839 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:07,712 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:07,770 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:07,773 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:07,773 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:07,774 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:08,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:09,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:10,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:10,727 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:10,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:10,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:11,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:52:11,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:52:11,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:52:11,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:11,715 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:13,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:13,681 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:13,682 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:13,683 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:13,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:13,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:15,717 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:16,603 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:16,605 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:16,606 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:16,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:16,608 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:16,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:19,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:19,550 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:19,550 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:19,551 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:19,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:19,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:21,721 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:22,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:22,484 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:22,485 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:22,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:22,486 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:22,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:22,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:25,723 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:52:26,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:52:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:52:26,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:26,322 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:26,322 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:26,324 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:26,324 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:52:26,724 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:27,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:28,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:29,030 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:29,032 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:29,032 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:29,033 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:29,726 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:31,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:31,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:31,956 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:31,957 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:31,958 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:32,728 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:33,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:33,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:34,879 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:34,882 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:34,882 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:34,883 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:35,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:37,732 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:37,808 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:37,810 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:37,811 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:37,812 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:38,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:39,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:39,812 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:40,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:40,804 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:40,804 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:40,805 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:41,067 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:52:41,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:52:41,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:52:41,736 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:43,737 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:43,753 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:43,755 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:43,756 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:43,757 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:44,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:45,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:45,758 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:46,692 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:46,694 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:46,694 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:46,695 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:46,739 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:49,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:49,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:49,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:49,633 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:49,741 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:49,741 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:51,633 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:51,742 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:52,568 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:52,571 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:52,571 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:52,572 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:52,743 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:55,531 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:55,533 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:55,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:55,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:55,745 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:55,745 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:52:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:52:56,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:52:56,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:52:56,252 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:52:57,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:52:57,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:52:58,459 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:52:58,461 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:52:58,461 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:52:58,462 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:52:58,747 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:01,395 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:01,397 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:01,398 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:01,399 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:01,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:01,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:02,400 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:03,750 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:04,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:04,355 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:04,356 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:04,357 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:04,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:05,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:07,288 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:07,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:07,292 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:07,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:07,753 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:08,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:09,754 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:10,228 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:10,230 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:10,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:10,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:10,755 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:11,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:53:11,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:53:11,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:53:11,755 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:13,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:13,162 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:13,162 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:13,164 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:13,757 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:14,164 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:15,758 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:16,115 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:16,117 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:16,117 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:16,118 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:16,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:17,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:19,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:19,046 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:19,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:19,047 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:19,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:20,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:21,762 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:21,980 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:21,982 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:21,982 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:21,983 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:22,763 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:23,763 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:25,010 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:25,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:25,013 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:25,014 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:25,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:26,014 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:53:26,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:53:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:53:26,294 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:53:27,766 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:27,957 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:27,959 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:27,959 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:27,960 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:28,767 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:29,768 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:30,911 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:30,913 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:30,913 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:30,914 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:31,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:31,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:33,770 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:33,839 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:33,842 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:33,842 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:33,843 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:34,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:35,772 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:36,793 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:36,818 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:36,819 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:36,820 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:37,773 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:37,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:39,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:39,726 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:39,726 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:39,727 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:39,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:39,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:41,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:53:41,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:53:41,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:53:41,776 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:42,656 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:42,660 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:42,660 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:42,661 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:42,777 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:43,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:45,594 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:45,596 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:45,597 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:45,598 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:45,778 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:45,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:47,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:48,529 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:48,531 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:48,532 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:48,533 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:48,780 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:49,533 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:51,464 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:51,466 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:51,466 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:51,467 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:51,782 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:51,782 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:53,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:54,401 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:54,403 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:54,403 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:54,404 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:54,784 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:55,404 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:53:55,785 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:53:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:53:56,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:53:56,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:53:56,341 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:53:57,329 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:53:57,334 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:53:57,334 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:53:57,335 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:53:57,786 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:53:59,788 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:00,284 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:00,286 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:00,286 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:00,287 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:00,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:01,288 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:01,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:03,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:03,227 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:03,228 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:03,229 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:03,791 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:05,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:06,155 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:06,157 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:06,158 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:06,159 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:06,793 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:07,159 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:07,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:09,172 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:09,174 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:09,174 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:09,175 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:09,795 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:11,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:54:11,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:54:11,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:54:11,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:12,101 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:12,103 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:12,103 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:12,105 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:12,797 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:13,105 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:13,797 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:15,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:15,050 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:15,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:15,052 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:15,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:17,800 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:17,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:17,977 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:17,977 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:17,978 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:18,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:18,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:19,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:20,900 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:20,902 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:20,902 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:20,903 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:21,803 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:23,804 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:23,826 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:23,828 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:23,829 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:23,830 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:24,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:24,830 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:25,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:54:26,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:54:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:54:26,304 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:54:26,763 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:26,765 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:26,765 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:26,766 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:26,806 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:29,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:29,699 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:29,699 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:29,701 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:29,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:29,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:30,701 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:31,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:32,625 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:32,628 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:32,628 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:32,629 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:32,811 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:35,556 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:35,559 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:35,559 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:35,560 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:35,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:35,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:36,561 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:37,814 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:38,507 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:38,509 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:38,509 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:38,510 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:38,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:39,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:41,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:54:41,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:54:41,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:54:41,442 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:41,444 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:41,445 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:41,446 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:41,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:42,446 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:43,817 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:44,369 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:44,371 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:44,372 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:44,373 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:44,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:45,819 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:47,297 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:47,299 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:47,300 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:47,302 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:47,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:48,302 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:49,821 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:50,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:50,322 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:50,322 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:50,323 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:50,822 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:51,823 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:53,249 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:53,251 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:53,252 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:53,253 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:53,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:54,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:54:55,825 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:54:56,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:54:56,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:54:56,187 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:56,303 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:54:56,303 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:56,303 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:56,305 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:56,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:54:57,827 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:54:59,139 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:54:59,141 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:54:59,141 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:54:59,143 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:54:59,828 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:00,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:01,829 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:02,087 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:02,090 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:02,090 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:02,091 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:02,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:03,831 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:05,049 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:05,051 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:05,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:05,053 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:05,832 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:06,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:07,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:07,999 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:08,002 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:08,002 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:08,003 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:08,835 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:09,835 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:10,926 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:10,928 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:10,928 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:10,929 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:11,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:55:11,068 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:55:11,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:55:11,279 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:11,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:13,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:13,870 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:13,872 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:13,872 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:13,873 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:14,839 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:15,839 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:16,798 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:16,800 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:16,801 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:16,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:16,802 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:16,840 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:19,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:19,726 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:19,727 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:19,728 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:19,842 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:19,842 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:21,843 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:22,689 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:22,691 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:22,692 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:22,692 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:22,693 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:22,844 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:25,624 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:25,626 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:25,626 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:25,628 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:25,846 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:25,846 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:26,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:55:26,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:55:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:55:26,259 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:55:27,847 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:28,260 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:28,571 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:28,574 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:28,574 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:28,575 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:28,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:29,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:31,519 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:31,521 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:31,522 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:31,523 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:31,850 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:33,524 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:33,851 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:34,569 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:34,571 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:34,571 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:34,572 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:34,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:35,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:37,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:37,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:37,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:37,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:37,854 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:39,506 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:39,855 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:40,473 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:40,475 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:40,475 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:40,477 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:40,856 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:41,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:55:41,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:55:41,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:55:41,856 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:43,415 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:43,416 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:43,417 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:43,419 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:43,858 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:45,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:45,858 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:46,360 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:46,363 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:46,363 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:46,365 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:46,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:47,860 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:49,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:49,295 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:49,296 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:49,297 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:49,861 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:51,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:51,862 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:52,241 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:52,243 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:52,243 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:52,245 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:52,863 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:53,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:55,192 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:55,194 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:55,194 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:55,196 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:55,865 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:56,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:55:56,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:55:56,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:55:56,263 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:55:57,264 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:55:57,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:55:58,127 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:55:58,129 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:55:58,131 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:55:58,133 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:55:58,867 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:55:59,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:01,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:01,057 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:01,057 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:01,058 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:01,869 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:03,059 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:03,870 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:03,981 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:03,983 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:03,984 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:03,985 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:04,871 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:05,872 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:06,909 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:06,911 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:06,911 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:06,912 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:07,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:08,913 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:09,848 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:09,851 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:09,851 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:09,852 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:09,875 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:09,875 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:11,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:56:11,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:56:11,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:56:11,876 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:12,770 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:12,772 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:12,772 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:12,773 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:12,877 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:14,774 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:15,795 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:15,797 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:15,798 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:15,799 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:15,879 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:15,879 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:17,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:18,723 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:18,725 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:18,726 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:18,727 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:18,881 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:20,727 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:21,652 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:21,655 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:21,655 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:21,656 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:21,883 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:21,884 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:23,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:24,607 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:24,609 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:24,610 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:24,611 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:24,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:25,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:26,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:56:26,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:56:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:56:26,330 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:56:26,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:27,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:27,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:27,557 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:27,558 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:27,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:29,890 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:30,489 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:30,492 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:30,492 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:30,493 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:30,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:31,494 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:31,893 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:33,428 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:33,430 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:33,430 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:33,431 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:33,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:35,895 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:36,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:36,367 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:36,367 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:36,369 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:36,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:37,369 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:37,897 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:39,308 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:39,310 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:39,310 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:39,311 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:39,899 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:41,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:56:41,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:56:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:56:41,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:42,243 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:42,245 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:42,246 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:42,247 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:42,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:43,247 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:43,902 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:45,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:45,179 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:45,179 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:45,180 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:45,903 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:47,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:48,105 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:48,107 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:48,108 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:48,109 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:48,905 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:49,109 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:49,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:51,032 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:51,035 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:51,035 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:51,036 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:51,907 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:53,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:53,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:53,961 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:53,961 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:53,962 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:54,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:54,963 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:56:55,910 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:56,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:56:56,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:56:56,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:56:56,299 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:56:56,887 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:56,889 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:56,889 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:56,891 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:56,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:56:59,819 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:56:59,821 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:56:59,822 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:56:59,823 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:56:59,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:56:59,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:00,823 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:01,914 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:02,843 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:02,845 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:02,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:02,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:02,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:05,781 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:05,784 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:05,784 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:05,785 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:05,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:05,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:06,785 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:07,918 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:08,710 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:08,712 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:08,712 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:08,713 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:08,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:09,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:11,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:57:11,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:57:11,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:57:11,683 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:11,686 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:11,686 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:11,687 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:11,921 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:12,688 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:13,922 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:14,619 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:14,622 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:14,622 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:14,623 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:14,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:15,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:17,561 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:17,563 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:17,563 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:17,564 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:17,924 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:18,565 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:19,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:20,493 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:20,497 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:20,497 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:20,498 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:20,926 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:21,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:23,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:23,468 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:23,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:23,471 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:23,929 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:24,471 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:25,930 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:26,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:57:26,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:57:26,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:57:26,247 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:57:26,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:26,456 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:26,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:26,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:26,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:27,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:29,388 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:29,390 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:29,391 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:29,392 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:29,933 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:30,392 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:31,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:32,326 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:32,328 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:32,328 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:32,330 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:32,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:33,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:35,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:35,298 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:35,298 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:35,300 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:35,937 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:36,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:37,938 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:38,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:38,222 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:38,222 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:38,224 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:38,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:39,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:41,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:57:41,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:57:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:57:41,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:41,240 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:41,240 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:41,242 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:41,941 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:42,242 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:43,941 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:44,174 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:44,177 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:44,177 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:44,179 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:44,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:45,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:47,099 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:47,101 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:47,102 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:47,103 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:47,945 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:48,104 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:49,945 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:50,028 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:50,030 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:50,031 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:50,032 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:50,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:51,947 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:52,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:52,998 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:52,998 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:53,000 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:53,948 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:54,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:57:55,939 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:55,941 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:55,942 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:55,943 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:55,950 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:55,950 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:56,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:57:56,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:57:56,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:57:56,291 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:57:57,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:57:58,884 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:57:58,886 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:57:58,887 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:57:58,888 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:57:58,952 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:57:59,888 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:01,837 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:01,839 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:01,839 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:01,841 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:01,953 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:01,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:03,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:04,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:04,787 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:04,787 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:04,788 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:04,955 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:05,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:05,956 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:07,710 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:07,712 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:07,713 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:07,714 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:07,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:09,959 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:10,654 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:10,657 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:10,657 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:10,658 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:10,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:11,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:58:11,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:58:11,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:58:11,243 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:11,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:13,598 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:13,600 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:13,601 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:13,602 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:13,962 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:15,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:16,548 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:16,550 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:16,551 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:16,551 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:16,552 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:16,964 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:17,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:19,483 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:19,485 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:19,485 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:19,487 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:19,966 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:21,967 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:22,431 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:22,433 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:22,434 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:22,434 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:22,435 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:22,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:23,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:25,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:25,366 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:25,366 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:25,372 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:25,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:26,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:58:26,069 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:58:26,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:58:26,273 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:58:27,971 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:28,274 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:28,390 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:28,393 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:28,393 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:28,394 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:28,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:29,973 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:31,338 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:31,340 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:31,340 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:31,342 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:31,974 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:33,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:33,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:34,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:34,269 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:34,269 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:34,270 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:34,976 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:35,977 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:37,215 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:37,218 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:37,218 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:37,219 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:37,978 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:39,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:39,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:40,166 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:40,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:40,169 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:40,170 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:40,980 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:41,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:58:41,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:58:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:58:41,981 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:43,094 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:43,096 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:43,097 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:43,098 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:43,982 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:45,098 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:45,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:46,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:46,062 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:46,063 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:46,064 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:46,984 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:47,985 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:48,989 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:48,991 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:48,991 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:48,993 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:49,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:50,993 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:51,922 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:51,924 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:51,925 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:51,926 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:51,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:51,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:53,989 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:54,857 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:54,859 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:54,860 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:54,861 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:54,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:55,991 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:58:56,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:58:56,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:58:56,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:58:56,307 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:58:56,307 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:58:57,810 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:58:57,812 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:58:57,813 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:58:57,814 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:58:57,995 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:58:59,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:00,761 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:00,764 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:00,764 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:00,766 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:00,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:01,766 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:01,998 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:03,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:03,700 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:03,700 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:03,701 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:04,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:06,001 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:06,632 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:06,634 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:06,635 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:06,636 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:07,002 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:07,637 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:08,003 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:09,570 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:09,572 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:09,572 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:09,573 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:10,005 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:11,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:59:11,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:59:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:59:12,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:12,595 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:12,597 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:12,598 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:12,599 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:13,007 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:13,600 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:14,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:15,531 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:15,533 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:15,533 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:15,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:16,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:18,011 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:18,486 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:18,489 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:18,490 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:18,491 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:19,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:19,492 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:20,013 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:21,440 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:21,444 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:21,445 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:21,446 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:22,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:24,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:24,367 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:24,370 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:24,370 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:24,371 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:25,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:25,372 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:26,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:59:26,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:59:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:59:26,269 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:59:27,288 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:27,291 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:27,291 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:27,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:28,019 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:30,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:30,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:30,255 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:30,255 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:30,256 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:31,022 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:31,257 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:32,022 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:33,183 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:33,185 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:33,186 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:33,187 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:34,024 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:36,025 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:36,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:36,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:36,120 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:36,121 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:37,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:37,122 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:38,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:39,065 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:39,067 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:39,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:39,068 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:40,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:41,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:59:41,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:59:41,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:59:41,990 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:41,993 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:41,993 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:41,994 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:42,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:42,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:42,995 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:44,032 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:44,942 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:44,944 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:44,945 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:44,946 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:45,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:46,034 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:47,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:47,931 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:47,932 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:47,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:48,035 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:48,933 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:50,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:50,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:50,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:50,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:50,869 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:51,038 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:52,038 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:53,892 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:53,917 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:53,918 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:53,919 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:54,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:54,920 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 20:59:56,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:56,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 20:59:56,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 20:59:56,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 20:59:56,342 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 20:59:56,831 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:56,833 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:56,834 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:56,835 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 20:59:57,042 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 20:59:58,043 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 20:59:59,800 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 20:59:59,802 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 20:59:59,803 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 20:59:59,804 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:00,045 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:00,804 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:02,046 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:02,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:02,732 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:02,732 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:02,733 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:03,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:04,048 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:05,683 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:05,685 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:05,686 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:05,687 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:06,050 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:06,687 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:08,051 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:08,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:08,611 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:08,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:08,612 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:09,052 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:10,053 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:11,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:00:11,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:00:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:00:11,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:11,538 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:11,539 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:11,540 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:12,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:12,541 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:14,057 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:14,475 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:14,477 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:14,477 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:14,479 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:15,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:16,059 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:17,402 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:17,404 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:17,405 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:17,406 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:18,060 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:18,407 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:20,061 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:20,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:20,330 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:20,330 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:20,331 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:21,063 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:22,063 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:23,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:23,273 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:23,273 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:23,275 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:24,065 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:24,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:26,066 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:00:26,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:00:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:00:26,208 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:26,297 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:00:26,297 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:26,297 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:26,299 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:27,067 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:28,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:29,140 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:29,142 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:29,143 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:29,144 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:30,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:30,145 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:32,071 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:32,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:32,086 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:32,087 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:32,088 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:33,072 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:34,073 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:35,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:35,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:35,012 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:35,013 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:35,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:36,013 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:36,075 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:38,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:38,039 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:38,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:38,040 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:38,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:40,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:40,972 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:40,977 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:40,977 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:40,979 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:41,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:00:41,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:00:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:00:41,079 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:41,336 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:42,079 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:43,906 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:43,908 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:43,909 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:43,910 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:44,081 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:46,082 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:46,843 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:46,845 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:46,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:46,845 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:46,847 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:47,083 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:48,084 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:49,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:49,796 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:49,797 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:49,798 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:50,085 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:52,086 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:52,747 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:52,749 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:52,750 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:52,750 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:52,751 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:53,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:54,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:55,678 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:55,680 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:55,680 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:55,681 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:56,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:00:56,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:00:56,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:00:56,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:00:56,245 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:00:58,091 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:00:58,245 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:00:58,612 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:00:58,615 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:00:58,615 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:00:58,616 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:00:59,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:00,093 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:01,542 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:01,544 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:01,545 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:01,546 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:02,095 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:03,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:04,096 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:04,490 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:04,492 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:04,493 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:04,494 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:05,097 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:06,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:07,426 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:07,428 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:07,429 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:07,430 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:08,099 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:09,430 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:10,100 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:10,380 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:10,382 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:10,382 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:10,383 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:11,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:01:11,070 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:01:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:01:11,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:12,102 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:13,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:13,326 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:13,326 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:13,327 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:14,104 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:15,328 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:16,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:16,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:16,266 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:16,266 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:16,267 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:17,106 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:18,107 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:19,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:19,298 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:19,298 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:19,299 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:20,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:21,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:22,109 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:22,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:22,228 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:22,228 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:22,229 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:23,111 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:24,111 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:25,162 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:25,164 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:25,164 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:25,165 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:26,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:01:26,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:01:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:01:26,113 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:26,347 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:01:26,347 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:28,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:28,115 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:28,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:28,116 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:28,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:28,116 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:30,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:31,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:31,054 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:31,055 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:31,056 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:31,118 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:32,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:32,119 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:33,981 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:33,984 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:33,984 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:33,987 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:34,120 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:36,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:36,928 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:36,930 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:36,931 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:36,932 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:37,122 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:37,932 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:38,122 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:39,878 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:39,880 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:39,881 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:39,882 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:40,124 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:01:41,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:01:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:01:42,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:42,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:42,803 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:42,803 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:42,805 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:43,126 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:43,805 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:44,126 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:45,728 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:45,730 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:45,731 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:45,732 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:46,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:48,129 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:48,660 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:48,662 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:48,662 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:48,664 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:49,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:49,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:50,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:51,626 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:51,627 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:51,628 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:51,629 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:52,132 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:54,133 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:54,555 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:54,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:54,558 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:54,559 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:55,134 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:01:55,559 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:01:56,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:01:56,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:01:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:01:56,134 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:01:56,276 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:01:57,482 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:01:57,484 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:01:57,485 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:01:57,487 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:01:58,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:00,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:00,470 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:00,475 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:00,476 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:00,477 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:01,138 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:01,478 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:02,138 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:03,405 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:03,407 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:03,408 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:03,409 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:04,140 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:06,141 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:06,347 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:06,349 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:06,350 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:06,351 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:07,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:07,351 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:08,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:09,277 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:09,279 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:09,280 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:09,281 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:10,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:11,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:02:11,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:02:11,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:02:12,144 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:12,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:12,214 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:12,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:12,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:12,700 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:12,700 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:13,146 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:14,146 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:15,838 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:15,840 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:15,840 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:15,841 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:16,147 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:17,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:18,149 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:18,770 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:18,774 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:18,774 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:18,775 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:19,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:20,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:21,719 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:21,721 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:21,722 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:21,723 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:22,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:23,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:24,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:24,648 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:24,651 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:24,651 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:24,652 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:25,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:26,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:02:26,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:02:26,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:02:26,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:26,267 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:02:27,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:27,611 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:27,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:27,613 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:28,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:29,614 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:30,157 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:30,557 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:30,559 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:30,560 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:30,561 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:31,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:32,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:33,487 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:33,489 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:33,489 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:33,491 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:34,160 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:35,491 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:36,161 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:36,421 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:36,423 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:36,424 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:36,425 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:37,162 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:38,162 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:39,356 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:39,359 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:39,359 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:39,360 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:40,164 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:02:41,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:02:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:02:41,316 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:42,165 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:42,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:42,301 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:42,302 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:42,303 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:43,166 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:44,167 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:45,323 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:45,325 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:45,326 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:45,327 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:46,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:46,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:48,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:48,259 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:48,262 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:48,262 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:48,265 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:49,170 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:50,171 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:51,198 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:51,200 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:51,201 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:51,202 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:52,172 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:52,202 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:54,131 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:54,133 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:54,133 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:54,134 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:54,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:54,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:56,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:02:56,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:02:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:02:56,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:02:56,765 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:02:57,070 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:02:57,076 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:02:57,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:02:57,078 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:02:57,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:02:58,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:02:58,176 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:00,033 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:00,035 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:00,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:00,037 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:00,177 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:02,178 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:02,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:02,973 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:02,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:02,974 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:03,179 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:03,975 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:04,180 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:05,912 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:05,914 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:05,914 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:05,916 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:06,181 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:08,182 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:08,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:08,845 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:08,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:08,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:09,183 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:09,847 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:10,184 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:11,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:03:11,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:03:11,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:03:11,775 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:11,777 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:11,777 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:11,778 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:12,185 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:14,186 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:14,706 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:14,708 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:14,709 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:14,710 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:15,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:15,710 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:16,188 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:17,637 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:17,640 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:17,640 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:17,641 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:18,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:20,190 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:20,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:20,576 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:20,577 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:20,578 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:21,192 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:21,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:22,192 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:23,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:23,541 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:23,541 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:23,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:24,194 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:26,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:03:26,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:03:26,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:03:26,195 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:26,314 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:03:26,480 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:26,482 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:26,482 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:26,483 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:27,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:27,484 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:28,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:29,408 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:29,411 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:29,411 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:29,412 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:30,199 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:32,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:32,451 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:32,454 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:32,454 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:32,456 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:33,201 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:33,456 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:34,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:35,413 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:35,415 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:35,415 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:35,417 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:36,203 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:38,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:38,375 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:38,377 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:38,377 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:38,378 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:39,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:39,379 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:40,207 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:03:41,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:03:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:03:41,337 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:41,339 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:41,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:41,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:42,208 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:44,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:44,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:44,271 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:44,271 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:44,272 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:45,211 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:45,273 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:46,212 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:47,219 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:47,221 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:47,222 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:47,223 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:48,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:48,214 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:50,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:50,145 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:50,145 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:50,147 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:50,215 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:51,147 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:52,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:53,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:53,065 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:53,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:53,067 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:53,218 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:54,218 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:55,987 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:55,989 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:55,990 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:55,991 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:56,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:03:56,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:03:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:03:56,220 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:03:56,333 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:03:56,334 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:03:58,221 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:03:58,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:03:58,918 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:03:58,918 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:03:58,919 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:03:59,222 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:00,223 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:01,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:01,849 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:01,849 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:01,849 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:01,851 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:02,225 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:04,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:04,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:04,789 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:04,789 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:04,790 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:05,227 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:06,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:07,745 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:07,747 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:07,748 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:07,748 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:07,749 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:08,229 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:10,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:10,702 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:10,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:10,704 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:10,705 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:04:11,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:04:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:04:11,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:12,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:13,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:13,633 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:13,635 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:13,636 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:13,637 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:14,234 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:16,235 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:16,652 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:16,654 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:16,654 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:16,656 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:17,237 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:18,237 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:18,656 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:19,606 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:19,609 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:19,609 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:19,610 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:20,239 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:22,240 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:22,557 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:22,560 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:22,560 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:22,561 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:23,241 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:24,242 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:24,562 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:25,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:25,491 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:25,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:25,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:26,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:04:26,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:04:26,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:04:26,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:26,351 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:04:28,245 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:28,413 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:28,415 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:28,416 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:28,417 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:29,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:30,247 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:30,418 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:31,369 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:31,371 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:31,372 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:31,373 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:32,248 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:34,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:34,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:34,333 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:34,334 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:34,335 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:35,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:36,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:36,335 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:37,266 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:37,269 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:37,269 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:37,271 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:38,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:38,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:40,204 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:40,207 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:40,207 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:40,208 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:40,254 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:41,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:04:41,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:04:41,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:04:42,255 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:42,287 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:43,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:43,136 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:43,136 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:43,137 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:43,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:44,257 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:46,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:46,075 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:46,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:46,077 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:46,258 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:48,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:48,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:49,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:49,004 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:49,005 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:49,006 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:49,260 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:50,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:51,929 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:51,931 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:51,931 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:51,933 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:52,262 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:53,933 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:04:54,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:54,852 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:54,854 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:54,855 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:54,856 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:55,264 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:56,071 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:04:56,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:04:56,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:04:56,264 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:04:56,320 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:04:57,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:04:57,823 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:04:57,824 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:04:57,825 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:04:58,266 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:04:59,826 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:00,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:00,848 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:00,850 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:00,851 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:00,852 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:01,268 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:02,269 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:03,777 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:03,779 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:03,779 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:03,781 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:04,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:05,782 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:06,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:06,699 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:06,702 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:06,702 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:06,704 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:07,272 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:08,273 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:09,646 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:09,649 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:09,649 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:09,650 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:10,274 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:05:11,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:05:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:05:11,337 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:12,275 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:12,574 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:12,577 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:12,577 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:12,578 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:13,277 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:14,277 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:15,519 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:15,522 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:15,522 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:15,523 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:16,279 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:16,524 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:18,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:18,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:18,443 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:18,443 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:18,444 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:19,282 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:20,282 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:21,362 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:21,365 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:21,365 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:21,366 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:22,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:22,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:22,367 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:24,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:24,319 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:24,320 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:24,321 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:25,286 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:05:26,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:05:26,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:05:26,245 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:05:26,287 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:27,254 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:27,256 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:27,257 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:27,258 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:27,288 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:28,258 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:28,289 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:30,210 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:30,212 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:30,212 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:30,214 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:30,290 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:32,292 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:33,177 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:33,180 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:33,180 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:33,181 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:33,293 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:34,182 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:34,293 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:36,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:36,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:36,114 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:36,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:36,295 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:38,296 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:39,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:39,067 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:39,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:39,068 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:39,297 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:40,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:40,298 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:41,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:05:41,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:05:41,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:05:42,111 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:42,137 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:42,137 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:42,138 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:42,300 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:44,301 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:45,058 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:45,060 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:45,061 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:45,062 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:45,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:46,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:46,303 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:47,992 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:47,994 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:47,994 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:47,996 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:48,304 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:50,305 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:50,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:50,949 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:50,949 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:50,950 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:51,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:51,951 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:52,307 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:53,916 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:53,919 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:53,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:53,920 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:54,308 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:56,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:05:56,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:05:56,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:05:56,257 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:05:56,309 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:56,890 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:56,892 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:56,893 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:56,894 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:05:57,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:05:57,895 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:05:58,311 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:05:59,828 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:05:59,831 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:05:59,831 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:05:59,832 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:00,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:02,313 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:02,798 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:02,800 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:02,801 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:02,802 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:03,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:03,803 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:04,315 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:05,724 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:05,726 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:05,726 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:05,727 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:06,316 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:08,317 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:08,654 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:08,657 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:08,657 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:08,658 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:09,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:09,659 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:10,319 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:06:11,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:06:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:06:11,623 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:11,625 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:11,626 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:11,627 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:12,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:14,321 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:14,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:14,549 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:14,550 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:14,551 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:15,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:15,552 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:16,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:17,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:17,484 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:17,484 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:17,485 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:18,324 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:20,325 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:20,407 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:20,410 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:20,410 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:20,411 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:21,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:21,412 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:22,327 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:23,339 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:23,341 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:23,342 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:23,343 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:24,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:24,328 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:06:26,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:06:26,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:06:26,309 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:06:26,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:26,367 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:26,367 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:26,369 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:27,330 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:27,369 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:28,331 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:29,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:29,304 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:29,304 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:29,305 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:29,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:30,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:32,226 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:32,228 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:32,228 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:32,230 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:32,334 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:33,230 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:34,335 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:35,150 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:35,152 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:35,153 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:35,154 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:35,335 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:36,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:38,095 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:38,097 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:38,097 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:38,100 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:38,337 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:39,100 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:40,338 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:41,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:41,040 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:41,040 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:41,041 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:41,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:06:41,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:06:41,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:06:41,339 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:42,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:43,987 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:43,993 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:43,993 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:43,994 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:44,341 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:44,995 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:46,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:46,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:46,922 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:46,922 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:46,923 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:47,343 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:48,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:49,858 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:49,860 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:49,861 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:49,862 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:50,345 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:50,862 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:52,346 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:52,785 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:52,787 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:52,787 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:52,788 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:53,347 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:54,348 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:55,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:55,711 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:55,712 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:55,713 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:56,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:06:56,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:06:56,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:06:56,302 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:06:56,302 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:06:56,349 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:06:58,350 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:06:58,647 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:06:58,649 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:06:58,650 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:06:58,651 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:06:59,351 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:00,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:01,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:01,585 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:01,585 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:01,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:01,587 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:02,353 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:04,354 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:04,529 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:04,531 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:04,532 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:04,533 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:05,355 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:06,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:07,534 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:07,570 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:07,572 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:07,573 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:07,574 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:08,357 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:10,358 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:10,498 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:10,501 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:10,501 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:10,502 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:11,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:07:11,072 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:07:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:07:11,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:12,360 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:13,305 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:13,435 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:13,437 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:13,437 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:13,438 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:14,361 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:14,362 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:16,383 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:16,385 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:16,385 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:16,387 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:17,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:18,364 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:18,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:19,306 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:19,308 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:19,308 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:19,309 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:19,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:20,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:22,247 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:22,248 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:22,249 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:22,250 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:22,367 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:24,251 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:24,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:25,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:25,192 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:25,192 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:25,193 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:25,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:26,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:07:26,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:07:26,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:07:26,330 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:07:26,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:28,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:28,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:28,119 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:28,121 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:28,371 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:30,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:30,372 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:31,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:31,045 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:31,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:31,046 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:31,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:32,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:33,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:33,999 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:33,999 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:34,000 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:34,375 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:36,001 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:36,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:36,924 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:36,926 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:36,927 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:36,928 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:37,377 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:38,378 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:39,873 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:39,876 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:39,876 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:39,878 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:40,380 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:41,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:07:41,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:07:41,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:07:41,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:42,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:42,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:42,848 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:42,849 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:42,850 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:43,382 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:44,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:45,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:45,791 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:45,792 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:45,793 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:46,384 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:46,793 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:48,385 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:48,719 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:48,721 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:48,722 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:48,723 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:49,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:50,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:51,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:51,711 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:51,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:51,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:52,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:52,713 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:07:54,390 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:54,643 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:54,646 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:54,646 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:54,647 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:55,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:07:56,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:07:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:07:56,251 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:07:56,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:07:57,575 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:07:57,577 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:07:57,578 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:07:57,579 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:07:58,394 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:07:58,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:00,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:00,498 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:00,501 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:00,501 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:00,502 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:01,397 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:02,397 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:03,442 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:03,445 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:03,445 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:03,446 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:04,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:04,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:04,447 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:06,361 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:06,363 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:06,364 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:06,365 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:06,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:08,402 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:09,292 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:09,294 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:09,295 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:09,296 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:09,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:10,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:10,404 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:11,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:08:11,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:08:11,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:08:12,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:12,222 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:12,223 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:12,224 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:12,405 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:14,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:15,148 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:15,151 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:15,151 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:15,152 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:15,408 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:16,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:16,408 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:18,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:18,081 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:18,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:18,082 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:18,410 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:20,411 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:21,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:21,012 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:21,013 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:21,014 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:21,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:22,014 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:22,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:23,973 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:23,975 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:23,975 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:23,977 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:24,415 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:26,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:08:26,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:08:26,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:08:26,341 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:08:26,416 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:26,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:26,917 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:26,917 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:26,919 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:27,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:27,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:28,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:29,837 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:29,840 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:29,840 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:29,841 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:30,420 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:32,421 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:32,831 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:32,833 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:32,833 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:32,835 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:33,423 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:33,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:34,423 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:35,752 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:35,755 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:35,755 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:35,756 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:36,425 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:38,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:38,714 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:38,717 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:38,717 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:38,718 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:39,427 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:39,719 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:40,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:08:41,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:08:41,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:08:41,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:41,663 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:41,664 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:41,665 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:42,429 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:44,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:44,586 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:44,588 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:44,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:44,590 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:45,431 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:45,591 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:46,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:47,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:47,519 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:47,519 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:47,521 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:48,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:48,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:50,448 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:50,450 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:50,450 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:50,451 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:51,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:51,452 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:52,437 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:53,386 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:53,389 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:53,389 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:53,390 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:53,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:54,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:08:56,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:08:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:08:56,293 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:08:56,322 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:56,324 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:56,324 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:56,325 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:56,440 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:08:57,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:08:58,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:08:59,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:08:59,258 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:08:59,258 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:08:59,260 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:08:59,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:00,444 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:02,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:02,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:02,191 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:02,193 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:02,445 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:03,193 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:04,447 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:05,118 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:05,120 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:05,121 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:05,122 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:05,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:06,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:08,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:08,048 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:08,048 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:08,049 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:08,450 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:09,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:10,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:10,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:10,980 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:10,981 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:10,982 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:11,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:09:11,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:09:11,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:09:11,452 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:12,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:13,911 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:13,913 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:13,914 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:13,915 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:14,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:14,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:16,469 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:16,903 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:16,906 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:16,906 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:16,907 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:17,470 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:18,471 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:19,836 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:19,838 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:19,839 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:19,840 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:20,473 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:20,840 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:22,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:22,781 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:22,783 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:22,784 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:22,785 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:23,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:24,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:25,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:25,727 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:25,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:25,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:26,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:09:26,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:09:26,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:09:26,253 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:09:26,253 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:26,478 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:28,479 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:28,660 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:28,663 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:28,663 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:28,664 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:29,480 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:30,481 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:31,586 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:31,588 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:31,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:31,589 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:31,590 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:32,483 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:32,483 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:34,518 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:34,520 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:34,521 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:34,522 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:35,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:36,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:37,452 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:37,454 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:37,455 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:37,455 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:37,456 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:37,487 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:38,487 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:40,374 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:40,376 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:40,376 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:40,378 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:40,489 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:09:41,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:09:41,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:09:42,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:43,291 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:43,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:43,301 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:43,302 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:43,303 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:43,491 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:44,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:46,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:46,231 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:46,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:46,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:46,493 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:48,495 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:49,165 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:49,166 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:49,167 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:49,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:49,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:49,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:50,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:52,098 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:52,102 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:52,102 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:52,103 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:52,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:54,499 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:55,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:55,038 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:55,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:55,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:09:55,040 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:55,500 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:09:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:09:56,073 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:09:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:09:56,264 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:09:56,500 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:09:57,988 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:09:57,990 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:09:57,991 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:09:57,992 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:09:58,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:00,503 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:00,967 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:00,970 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:00,970 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:00,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:00,972 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:01,504 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:02,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:03,903 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:03,905 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:03,905 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:03,907 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:04,506 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:06,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:06,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:06,848 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:06,849 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:06,849 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:06,850 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:07,508 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:08,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:09,819 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:09,821 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:09,822 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:09,823 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:10,510 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:11,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:10:11,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:10:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:10:12,248 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:12,511 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:12,763 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:12,765 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:12,765 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:12,766 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:13,512 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:14,513 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:15,698 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:15,700 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:15,701 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:15,702 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:16,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:16,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:17,703 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:18,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:18,637 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:18,637 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:18,638 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:19,516 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:20,517 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:21,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:21,584 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:21,585 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:21,586 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:22,518 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:22,518 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:23,587 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:24,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:24,506 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:24,506 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:24,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:24,519 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:26,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:10:26,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:10:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:10:26,333 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:10:26,520 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:27,425 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:27,428 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:27,429 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:27,430 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:27,521 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:28,522 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:29,430 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:30,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:30,368 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:30,368 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:30,369 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:30,523 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:32,524 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:33,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:33,292 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:33,292 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:33,294 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:33,525 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:34,526 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:35,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:36,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:36,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:36,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:36,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:36,527 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:38,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:39,146 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:39,148 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:39,148 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:39,149 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:39,529 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:40,530 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:41,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:10:41,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:10:41,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:10:41,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:42,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:42,087 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:42,088 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:42,089 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:42,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:44,532 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:45,062 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:45,064 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:45,065 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:45,066 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:45,533 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:46,533 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:47,066 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:48,005 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:48,007 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:48,008 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:48,009 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:48,535 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:50,536 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:50,943 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:50,945 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:50,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:50,947 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:51,537 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:52,537 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:52,948 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:53,892 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:53,895 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:53,895 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:53,896 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:54,538 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:56,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:10:56,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:10:56,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:10:56,345 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:10:56,539 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:56,820 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:56,822 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:56,823 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:56,824 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:10:57,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:10:58,541 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:10:58,825 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:10:59,748 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:10:59,750 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:10:59,750 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:10:59,752 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:00,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:00,542 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:02,691 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:02,694 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:02,695 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:02,696 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:03,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:04,544 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:04,696 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:05,625 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:05,627 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:05,627 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:05,629 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:06,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:06,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:08,566 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:08,569 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:08,569 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:08,570 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:09,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:10,549 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:10,571 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:11:11,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:11:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:11:11,494 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:11,496 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:11,497 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:11,498 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:11,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:12,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:14,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:14,443 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:14,444 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:14,445 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:14,552 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:16,445 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:16,553 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:17,386 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:17,388 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:17,389 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:17,390 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:17,554 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:18,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:20,306 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:20,309 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:20,309 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:20,310 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:20,556 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:22,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:22,557 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:23,238 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:23,241 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:23,241 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:23,242 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:23,559 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:24,559 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:26,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:11:26,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:11:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:11:26,188 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:26,306 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:26,307 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:26,308 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:26,308 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:11:26,561 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:28,309 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:28,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:29,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:29,214 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:29,214 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:29,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:29,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:30,565 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:32,141 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:32,144 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:32,145 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:32,152 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:32,567 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:34,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:34,568 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:35,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:35,110 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:35,111 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:35,112 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:35,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:36,569 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:38,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:38,040 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:38,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:38,042 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:38,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:40,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:40,572 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:40,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:40,972 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:40,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:40,974 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:41,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:11:41,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:11:41,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:11:41,573 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:42,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:43,903 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:43,905 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:43,906 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:43,907 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:44,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:45,908 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:46,576 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:46,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:46,849 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:46,849 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:46,850 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:47,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:48,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:49,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:49,791 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:49,792 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:49,793 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:50,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:50,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:51,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:52,719 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:52,722 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:52,722 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:52,723 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:53,582 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:54,582 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:55,639 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:55,641 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:55,642 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:55,643 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:56,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:11:56,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:11:56,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:11:56,344 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:11:56,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:11:56,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:11:57,344 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:11:58,569 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:11:58,571 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:11:58,572 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:11:58,573 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:11:58,586 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:00,587 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:01,515 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:01,518 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:01,518 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:01,519 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:01,588 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:01,858 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:02,589 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:03,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:04,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:04,970 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:04,973 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:04,973 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:04,974 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:05,591 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:06,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:07,934 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:07,937 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:07,938 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:07,939 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:08,593 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:08,940 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:10,594 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:10,864 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:10,866 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:10,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:10,868 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:12:11,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:12:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:12:11,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:12,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:13,857 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:13,859 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:13,859 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:13,861 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:14,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:14,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:14,861 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:16,807 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:16,809 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:16,810 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:16,811 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:17,600 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:18,601 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:19,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:19,732 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:19,732 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:19,733 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:20,603 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:20,603 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:20,733 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:22,649 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:22,652 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:22,653 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:22,654 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:23,605 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:24,606 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:25,601 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:25,604 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:25,604 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:25,605 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:25,607 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:26,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:12:26,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:12:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:12:26,340 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:12:26,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:26,608 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:28,523 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:28,526 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:28,526 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:28,527 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:28,609 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:30,611 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:31,456 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:31,459 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:31,459 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:31,459 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:31,460 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:31,612 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:32,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:34,384 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:34,387 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:34,387 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:34,388 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:34,614 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:36,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:37,310 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:37,312 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:37,312 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:37,312 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:37,313 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:37,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:38,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:40,236 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:40,238 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:40,238 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:40,240 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:40,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:41,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:12:41,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:12:41,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:12:42,620 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:43,164 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:43,166 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:43,167 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:43,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:43,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:43,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:44,622 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:46,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:46,110 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:46,110 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:46,111 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:46,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:48,625 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:49,036 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:49,038 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:49,039 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:49,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:49,040 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:49,626 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:50,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:51,966 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:51,968 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:51,968 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:51,969 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:52,628 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:54,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:54,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:54,893 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:54,894 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:54,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:12:54,895 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:55,631 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:56,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:12:56,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:12:56,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:12:56,324 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:12:56,631 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:12:57,922 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:12:57,924 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:12:57,925 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:12:57,926 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:12:58,633 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:12:59,927 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:00,634 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:00,854 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:00,857 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:00,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:00,858 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:01,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:02,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:03,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:03,789 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:03,790 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:03,791 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:04,638 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:04,639 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:05,792 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:06,716 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:06,718 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:06,719 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:06,720 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:07,641 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:08,641 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:09,647 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:09,650 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:09,650 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:09,651 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:10,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:10,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:11,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:13:11,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:13:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:13:11,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:12,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:12,581 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:12,581 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:12,584 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:12,645 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:14,646 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:15,526 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:15,528 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:15,529 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:15,530 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:15,647 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:16,530 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:16,648 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:18,462 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:18,464 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:18,465 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:18,466 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:18,649 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:20,651 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:21,395 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:21,397 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:21,398 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:21,399 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:21,652 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:22,400 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:22,653 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:24,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:24,325 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:24,326 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:24,327 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:24,654 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:26,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:13:26,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:13:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:13:26,292 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:13:26,655 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:27,277 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:27,280 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:27,280 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:27,281 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:27,656 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:28,282 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:28,657 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:30,210 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:30,212 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:30,212 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:30,214 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:30,659 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:32,660 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:33,129 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:33,131 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:33,132 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:33,133 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:33,661 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:34,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:34,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:36,048 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:36,051 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:36,051 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:36,052 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:36,663 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:38,665 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:38,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:38,976 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:38,977 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:38,978 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:39,666 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:39,978 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:40,667 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:41,074 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:13:41,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:13:41,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:13:42,001 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:42,004 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:42,004 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:42,005 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:42,668 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:44,669 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:44,957 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:44,959 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:44,959 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:44,961 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:45,670 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:45,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:46,671 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:47,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:47,917 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:47,917 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:47,919 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:48,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:48,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:50,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:50,844 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:50,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:50,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:51,675 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:51,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:52,675 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:53,775 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:53,777 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:53,777 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:53,778 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:54,677 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:54,677 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:13:56,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:13:56,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:13:56,339 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:13:56,737 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:56,740 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:56,741 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:56,742 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:57,679 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:13:57,742 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:13:58,679 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:13:59,674 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:13:59,677 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:13:59,677 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:13:59,678 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:13:59,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:00,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:02,616 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:02,619 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:02,619 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:02,620 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:02,682 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:03,621 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:04,683 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:05,538 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:05,540 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:05,540 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:05,542 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:05,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:06,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:08,461 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:08,463 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:08,464 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:08,465 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:08,686 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:09,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:10,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:11,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:14:11,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:14:11,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:14:11,419 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:11,421 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:11,422 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:11,423 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:11,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:12,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:14,358 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:14,360 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:14,361 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:14,363 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:14,690 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:15,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:16,691 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:17,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:17,295 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:17,296 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:17,297 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:17,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:18,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:20,219 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:20,221 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:20,222 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:20,223 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:20,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:21,224 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:22,695 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:23,312 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:23,315 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:23,315 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:23,316 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:23,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:24,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:26,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:14:26,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:14:26,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:14:26,247 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:26,289 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:14:26,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:26,290 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:26,290 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:26,291 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:26,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:28,699 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:29,195 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:29,197 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:29,197 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:29,199 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:29,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:30,701 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:32,125 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:32,127 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:32,127 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:32,128 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:32,129 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:32,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:34,704 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:35,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:35,054 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:35,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:35,055 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:35,705 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:36,706 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:37,986 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:37,988 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:37,989 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:37,989 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:37,990 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:38,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:38,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:40,921 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:40,923 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:40,924 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:40,925 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:41,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:14:41,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:14:41,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:14:41,710 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:42,711 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:43,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:43,853 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:43,855 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:43,856 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:43,857 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:44,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:44,713 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:46,796 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:46,799 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:46,799 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:46,800 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:47,715 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:48,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:48,801 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:49,743 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:49,746 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:49,747 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:49,748 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:50,717 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:50,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:52,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:52,683 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:52,683 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:52,685 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:52,719 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:54,685 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:14:54,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:55,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:55,610 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:55,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:55,612 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:55,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:14:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:14:56,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:14:56,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:14:56,318 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:14:56,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:14:58,542 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:14:58,544 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:14:58,545 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:14:58,546 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:14:58,724 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:00,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:00,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:01,480 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:01,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:01,483 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:01,484 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:01,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:02,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:04,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:04,457 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:04,457 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:04,458 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:04,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:06,459 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:06,730 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:07,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:07,507 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:07,507 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:07,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:07,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:08,732 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:10,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:10,443 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:10,444 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:10,445 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:10,734 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:11,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:15:11,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:15:11,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:15:12,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:12,735 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:13,379 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:13,381 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:13,381 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:13,383 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:13,736 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:14,737 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:16,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:16,313 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:16,314 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:16,315 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:16,739 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:17,315 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:18,740 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:19,264 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:19,266 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:19,267 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:19,268 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:19,741 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:20,741 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:22,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:22,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:22,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:22,217 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:22,743 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:23,217 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:24,744 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:25,148 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:25,150 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:25,151 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:25,152 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:25,745 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:26,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:15:26,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:15:26,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:15:26,304 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:15:26,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:28,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:28,087 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:28,088 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:28,089 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:28,748 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:28,748 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:29,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:31,029 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:31,032 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:31,032 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:31,033 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:31,750 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:32,750 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:33,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:33,963 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:33,964 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:33,965 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:34,752 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:34,752 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:34,965 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:36,911 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:36,914 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:36,914 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:36,915 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:37,754 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:38,755 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:39,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:39,845 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:39,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:39,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:40,757 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:40,757 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:40,846 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:41,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:15:41,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:15:41,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:15:42,776 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:42,778 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:42,779 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:42,780 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:43,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:44,760 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:45,705 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:45,708 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:45,708 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:45,709 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:45,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:46,710 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:46,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:48,740 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:48,742 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:48,743 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:48,744 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:48,763 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:50,764 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:51,662 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:51,664 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:51,665 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:51,666 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:51,765 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:52,666 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:52,766 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:54,600 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:54,602 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:54,603 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:54,605 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:54,768 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:15:56,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:15:56,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:15:56,262 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:15:56,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:15:57,555 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:15:57,557 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:15:57,558 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:15:57,559 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:15:57,770 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:15:58,559 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:15:58,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:00,480 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:00,482 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:00,482 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:00,483 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:00,772 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:02,773 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:03,415 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:03,417 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:03,417 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:03,418 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:03,774 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:04,419 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:04,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:06,339 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:06,341 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:06,342 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:06,343 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:06,777 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:08,778 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:09,264 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:09,267 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:09,267 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:09,268 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:09,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:10,269 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:10,780 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:11,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:16:11,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:16:11,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:16:12,206 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:12,208 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:12,209 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:12,210 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:12,782 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:14,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:15,134 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:15,137 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:15,137 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:15,138 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:15,784 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:16,139 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:16,785 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:18,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:18,065 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:18,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:18,067 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:18,787 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:18,787 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:21,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:21,013 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:21,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:21,015 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:21,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:22,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:22,790 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:23,943 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:23,945 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:23,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:23,947 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:24,791 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:24,791 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:26,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:16:26,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:16:26,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:16:26,317 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:16:26,890 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:26,892 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:26,892 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:26,894 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:27,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:27,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:28,794 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:29,815 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:29,817 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:29,818 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:29,819 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:30,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:30,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:32,835 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:32,837 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:32,838 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:32,839 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:33,798 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:33,840 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:34,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:35,764 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:35,766 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:35,767 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:35,768 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:35,800 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:36,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:38,704 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:38,706 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:38,706 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:38,707 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:38,803 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:39,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:40,804 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:41,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:16:41,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:16:41,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:16:41,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:41,632 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:41,632 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:41,633 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:41,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:42,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:44,551 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:44,554 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:44,554 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:44,556 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:44,807 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:45,556 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:46,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:47,476 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:47,480 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:47,480 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:47,481 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:47,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:48,810 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:50,404 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:50,406 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:50,407 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:50,408 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:50,812 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:51,408 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:52,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:53,360 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:53,362 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:53,362 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:53,363 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:53,814 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:54,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:56,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:16:56,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:16:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:16:56,288 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:56,345 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:16:56,346 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:56,347 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:56,348 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:56,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:16:57,349 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:16:58,817 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:16:59,225 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:16:59,227 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:16:59,228 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:16:59,229 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:16:59,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:00,819 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:02,152 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:02,155 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:02,155 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:02,156 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:02,821 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:02,821 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:03,157 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:05,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:05,093 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:05,093 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:05,094 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:05,823 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:06,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:08,035 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:08,037 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:08,038 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:08,039 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:08,825 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:08,825 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:09,039 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:10,985 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:10,987 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:10,988 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:10,989 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:17:11,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:17:11,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:17:11,827 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:12,828 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:13,962 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:13,965 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:13,965 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:13,967 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:14,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:14,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:14,967 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:16,909 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:16,911 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:16,912 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:16,913 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:17,832 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:18,833 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:19,851 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:19,879 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:19,880 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:19,881 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:20,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:20,835 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:20,881 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:22,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:22,789 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:22,789 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:22,790 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:22,836 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:24,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:25,729 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:25,731 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:25,731 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:25,733 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:25,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:17:26,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:17:26,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:17:26,311 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:17:26,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:26,839 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:28,655 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:28,657 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:28,658 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:28,659 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:28,841 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:30,842 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:31,576 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:31,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:31,579 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:31,579 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:31,580 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:31,843 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:32,844 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:34,498 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:34,500 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:34,500 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:34,502 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:34,845 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:36,846 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:37,425 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:37,428 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:37,428 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:37,428 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:37,429 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:37,847 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:38,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:40,354 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:40,356 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:40,357 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:40,358 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:40,850 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:41,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:17:41,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:17:41,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:17:42,851 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:43,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:43,295 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:43,296 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:43,296 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:43,297 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:43,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:44,853 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:46,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:46,232 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:46,232 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:46,233 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:46,854 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:46,854 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:49,155 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:49,160 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:49,160 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:49,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:49,161 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:49,857 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:50,857 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:52,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:52,081 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:52,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:52,082 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:52,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:52,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:55,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:55,006 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:55,006 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:55,007 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:17:55,008 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:55,861 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:17:56,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:17:56,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:17:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:17:56,354 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:17:56,862 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:58,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:17:58,021 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:17:58,022 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:17:58,023 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:17:58,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:17:58,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:00,024 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:00,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:00,949 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:00,949 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:00,950 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:01,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:02,867 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:03,880 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:03,882 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:03,882 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:03,884 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:04,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:04,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:05,884 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:06,816 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:06,818 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:06,818 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:06,819 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:06,870 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:08,871 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:09,773 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:09,776 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:09,777 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:09,778 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:09,872 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:10,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:18:11,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:18:11,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:18:12,703 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:15,625 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:16,201 ERROR SenderThread:10151 [internal_api.py:execute():373] 502 response executing GraphQL. +2024-08-03 21:18:16,201 ERROR SenderThread:10151 [internal_api.py:execute():374] + + +502 Server Error + + +

Error: Server Error

+

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

+

+ + +2024-08-03 21:18:17,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: keepalive +2024-08-03 21:18:17,623 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:17,623 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:17,623 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:17,625 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:17,625 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:17,625 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:17,626 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:17,877 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:18,544 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:18,547 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:18,547 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:18,548 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:18,878 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:18,878 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:20,879 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:21,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:21,469 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:21,469 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:21,471 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:21,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:22,881 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:23,471 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:24,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:24,389 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:24,390 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:24,391 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:24,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:18:26,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:18:26,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:18:26,883 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:27,308 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:28,666 ERROR SenderThread:10151 [internal_api.py:execute():373] 502 response executing GraphQL. +2024-08-03 21:18:28,666 ERROR SenderThread:10151 [internal_api.py:execute():374] + + +502 Server Error + + +

Error: Server Error

+

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

+

+ + +2024-08-03 21:18:29,946 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:18:29,947 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:29,947 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:29,947 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:29,948 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:30,229 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:30,231 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:30,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:30,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:30,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:30,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:33,156 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:33,158 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:33,159 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:33,160 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:33,888 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:34,889 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:35,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:36,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:36,083 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:36,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:36,084 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:36,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:36,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:39,005 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:39,008 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:39,008 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:39,009 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:39,893 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:40,893 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:41,010 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:41,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:18:41,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:18:41,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:18:41,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:41,948 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:41,948 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:41,949 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:42,895 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:42,895 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:44,945 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:44,947 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:44,947 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:44,948 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:45,897 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:46,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:46,949 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:47,871 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:47,873 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:47,873 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:47,876 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:47,899 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:48,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:50,792 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:50,794 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:50,795 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:50,796 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:50,901 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:52,796 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:52,902 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:53,713 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:53,715 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:53,716 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:53,717 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:53,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:54,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:56,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:18:56,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:18:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:18:56,286 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:18:56,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:56,636 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:56,636 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:56,637 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:56,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:18:58,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:18:58,907 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:18:59,560 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:18:59,563 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:18:59,564 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:18:59,565 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:18:59,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:00,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:02,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:02,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:02,483 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:02,484 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:02,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:04,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:04,912 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:05,404 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:05,406 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:05,407 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:05,408 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:05,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:06,914 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:08,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:08,330 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:08,330 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:08,331 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:08,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:08,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:10,332 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:11,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:19:11,076 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:19:11,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:19:11,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:11,349 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:11,349 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:11,350 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:11,917 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:12,918 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:14,180 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:14,182 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:14,182 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:14,183 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:14,920 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:14,920 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:16,184 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:17,110 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:17,112 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:17,113 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:17,114 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:17,922 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:18,922 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:20,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:20,040 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:20,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:20,042 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:20,924 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:20,924 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:22,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:22,966 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:22,968 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:22,969 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:22,970 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:23,926 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:24,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:25,905 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:25,908 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:25,908 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:25,909 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:25,928 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:26,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:19:26,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:19:26,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:19:26,294 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:19:26,928 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:27,295 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:28,942 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:28,944 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:28,944 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:28,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:29,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:30,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:31,866 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:31,868 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:31,869 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:31,870 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:31,932 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:32,871 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:32,932 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:34,795 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:34,797 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:34,798 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:34,799 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:34,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:36,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:37,743 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:37,745 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:37,745 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:37,746 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:37,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:38,747 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:38,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:40,670 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:40,672 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:40,672 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:40,674 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:40,938 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:41,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:19:41,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:19:41,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:19:42,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:43,609 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:43,610 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:43,611 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:43,612 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:43,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:44,653 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:44,940 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:46,540 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:46,542 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:46,543 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:46,544 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:46,942 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:48,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:49,473 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:49,475 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:49,475 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:49,477 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:49,944 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:50,477 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:50,944 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:52,419 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:52,421 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:52,422 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:52,423 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:52,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:52,946 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:55,348 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:55,350 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:55,351 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:55,352 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:55,948 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:19:56,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:19:56,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:19:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:19:56,353 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:19:56,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:19:56,949 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:58,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:19:58,277 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:19:58,277 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:19:58,278 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:19:58,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:19:58,951 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:01,202 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:01,204 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:01,205 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:01,206 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:01,953 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:02,206 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:02,954 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:04,129 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:04,132 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:04,132 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:04,134 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:04,955 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:04,955 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:07,108 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:07,110 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:07,111 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:07,112 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:07,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:08,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:08,958 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:10,043 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:10,045 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:10,045 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:10,047 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:10,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:10,960 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:11,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:20:11,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:20:11,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:20:13,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:13,078 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:13,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:13,079 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:13,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:13,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:14,963 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:16,021 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:16,026 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:16,027 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:16,028 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:16,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:16,965 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:18,940 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:18,943 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:18,943 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:18,944 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:18,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:18,966 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:20,967 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:21,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:21,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:21,870 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:21,872 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:21,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:22,968 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:24,803 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:24,805 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:24,806 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:24,806 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:24,807 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:24,970 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:26,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:20:26,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:20:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:20:26,300 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:20:26,971 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:27,746 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:27,747 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:27,748 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:27,749 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:27,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:28,972 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:30,678 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:30,681 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:30,681 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:30,682 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:30,682 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:30,974 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:32,975 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:33,627 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:33,629 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:33,629 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:33,631 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:33,976 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:34,977 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:36,589 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:36,591 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:36,591 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:36,592 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:36,593 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:36,978 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:38,979 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:39,559 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:39,562 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:39,563 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:39,565 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:39,980 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:40,980 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:41,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:20:41,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:20:41,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:20:42,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:42,496 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:42,498 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:42,498 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:42,499 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:42,982 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:44,983 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:45,431 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:45,433 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:45,433 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:45,434 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:45,984 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:46,984 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:47,435 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:48,364 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:48,366 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:48,367 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:48,368 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:48,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:48,986 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:51,301 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:51,303 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:51,304 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:51,305 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:51,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:52,988 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:53,306 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:54,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:54,343 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:54,343 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:54,344 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:54,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:20:54,990 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:56,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:20:56,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:20:56,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:20:56,341 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:20:57,270 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:20:57,271 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:20:57,272 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:20:57,273 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:20:57,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:20:58,313 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:20:58,992 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:00,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:00,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:00,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:00,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:00,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:00,994 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:03,139 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:03,141 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:03,141 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:03,143 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:03,995 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:04,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:04,996 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:06,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:06,074 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:06,074 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:06,075 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:06,997 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:06,997 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:09,000 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:09,002 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:09,002 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:09,003 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:09,999 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:10,004 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:11,000 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:11,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:21:11,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:21:11,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:21:11,951 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:11,952 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:11,952 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:11,954 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:12,001 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:13,001 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:14,891 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:14,893 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:14,893 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:14,896 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:15,002 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:15,896 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:17,004 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:17,833 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:17,835 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:17,836 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:17,837 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:18,005 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:19,006 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:20,795 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:20,798 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:20,798 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:20,799 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:21,007 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:21,800 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:23,008 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:23,728 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:23,730 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:23,730 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:23,732 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:24,009 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:25,010 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:26,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:21:26,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:21:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:21:26,276 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:21:26,684 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:26,685 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:26,685 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:26,686 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:27,012 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:27,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:29,013 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:29,645 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:29,649 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:29,649 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:29,653 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:30,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:31,014 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:32,580 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:32,582 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:32,583 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:32,584 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:33,016 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:33,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:35,017 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:35,521 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:35,524 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:35,524 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:35,525 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:36,018 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:37,018 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:38,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:38,541 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:38,542 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:38,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:39,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:39,020 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:39,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:41,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:21:41,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:21:41,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:21:41,471 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:41,514 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:41,515 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:41,516 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:42,021 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:43,022 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:44,425 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:44,427 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:44,427 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:44,428 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:45,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:45,023 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:45,469 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:47,371 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:47,373 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:47,374 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:47,375 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:48,025 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:49,025 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:50,302 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:50,305 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:50,305 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:50,306 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:50,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:50,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:51,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:51,027 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:53,028 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:53,817 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:53,820 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:53,820 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:53,821 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:54,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:54,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:55,029 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:55,822 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:21:56,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:21:56,077 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:21:56,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:21:56,269 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:21:56,756 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:56,758 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:56,758 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:56,759 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:21:57,031 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:21:59,032 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:21:59,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:21:59,695 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:21:59,695 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:21:59,697 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:00,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:01,033 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:01,697 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:02,636 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:02,638 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:02,638 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:02,639 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:03,034 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:05,035 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:05,587 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:05,589 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:05,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:05,591 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:06,036 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:07,037 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:07,591 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:08,546 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:08,549 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:08,549 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:08,550 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:09,038 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:09,038 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:11,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:22:11,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:22:11,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:22:11,474 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:11,475 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:11,475 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:11,477 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:12,040 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:13,041 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:13,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:14,399 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:14,401 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:14,402 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:14,403 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:15,042 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:15,042 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:17,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:17,343 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:17,343 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:17,345 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:18,044 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:19,045 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:19,345 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:20,410 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:20,413 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:20,413 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:20,414 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:21,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:21,047 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:23,338 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:23,340 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:23,340 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:23,342 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:24,049 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:25,049 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:25,343 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:26,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:22:26,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:22:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:22:26,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:26,351 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:22:26,352 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:26,352 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:26,353 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:27,052 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:27,053 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:29,203 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:29,205 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:29,205 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:29,206 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:30,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:31,056 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:31,245 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:32,134 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:32,138 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:32,138 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:32,139 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:33,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:33,058 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:35,072 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:35,075 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:35,075 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:35,076 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:36,061 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:37,061 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:37,077 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:37,999 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:38,002 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:38,002 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:38,003 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:38,062 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:39,063 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:40,926 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:40,929 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:40,929 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:40,930 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:41,065 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:41,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:22:41,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:22:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:22:42,353 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:43,066 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:43,855 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:43,857 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:43,857 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:43,858 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:44,067 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:45,068 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:46,799 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:46,802 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:46,802 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:46,803 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:47,069 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:47,803 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:49,070 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:49,726 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:49,729 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:49,729 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:49,730 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:50,072 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:51,072 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:52,676 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:52,702 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:52,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:52,704 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:53,074 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:53,705 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:22:55,075 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:55,619 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:55,622 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:55,622 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:55,623 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:56,076 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:22:56,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:22:56,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:22:56,340 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:22:57,077 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:58,551 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:22:58,553 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:22:58,553 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:22:58,554 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:22:59,078 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:22:59,079 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:22:59,593 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:01,479 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:01,481 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:01,481 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:01,483 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:02,081 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:03,081 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:04,495 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:04,497 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:04,497 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:04,499 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:05,083 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:05,083 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:05,499 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:07,421 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:07,424 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:07,424 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:07,425 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:08,085 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:09,086 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:10,347 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:10,349 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:10,349 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:10,350 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:11,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:23:11,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:23:11,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:23:11,087 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:11,088 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:11,250 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:13,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:13,288 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:13,288 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:13,289 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:14,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:15,090 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:16,213 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:16,215 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:16,215 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:16,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:17,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:17,092 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:17,217 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:19,138 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:19,140 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:19,141 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:19,142 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:20,094 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:21,095 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:22,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:22,067 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:22,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:22,068 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:22,096 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:23,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:23,097 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:24,991 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:24,993 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:24,994 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:24,995 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:25,098 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:26,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:23:26,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:23:26,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:23:26,305 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:23:27,099 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:27,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:27,955 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:27,955 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:27,956 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:28,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:28,997 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:29,101 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:30,914 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:30,916 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:30,917 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:30,918 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:31,103 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:33,104 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:33,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:33,843 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:33,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:33,845 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:34,105 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:34,845 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:35,106 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:36,771 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:36,774 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:36,774 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:36,775 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:37,108 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:39,109 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:39,721 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:39,723 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:39,723 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:39,725 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:40,110 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:40,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:41,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:23:41,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:23:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:23:41,111 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:42,643 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:42,644 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:42,645 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:42,646 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:43,112 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:43,112 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:45,586 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:45,588 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:45,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:45,590 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:46,114 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:46,591 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:47,115 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:48,617 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:48,619 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:48,619 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:48,621 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:49,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:49,117 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:51,560 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:51,563 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:51,563 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:51,564 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:52,119 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:52,565 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:53,119 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:54,512 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:54,514 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:54,514 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:54,515 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:55,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:23:55,121 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:23:56,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:23:56,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:23:56,341 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:23:57,440 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:23:57,442 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:23:57,442 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:23:57,443 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:23:58,123 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:23:58,485 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:23:59,123 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:00,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:00,365 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:00,366 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:00,367 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:01,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:01,125 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:03,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:03,291 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:03,291 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:03,293 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:04,127 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:04,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:05,128 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:06,212 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:06,214 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:06,214 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:06,216 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:07,129 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:07,130 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:09,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:09,134 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:09,134 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:09,135 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:10,131 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:10,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:11,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:24:11,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:24:11,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:24:11,132 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:12,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:12,053 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:12,053 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:12,054 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:12,133 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:13,134 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:14,972 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:14,974 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:14,974 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:14,976 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:15,135 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:15,976 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:17,136 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:17,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:17,896 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:17,896 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:17,898 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:18,137 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:19,138 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:20,816 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:20,818 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:20,818 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:20,819 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:21,139 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:21,820 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:23,140 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:23,737 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:23,743 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:23,743 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:23,744 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:24,141 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:25,142 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:26,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:24:26,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:24:26,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:24:26,275 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:24:26,656 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:26,658 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:26,658 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:26,660 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:27,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:27,143 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:27,701 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:29,577 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:29,579 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:29,580 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:29,581 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:30,145 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:31,146 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:32,588 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:32,590 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:32,591 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:32,592 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:33,147 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:33,148 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:33,593 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:35,508 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:35,510 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:35,510 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:35,512 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:36,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:37,150 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:38,428 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:38,431 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:38,431 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:38,432 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:39,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:39,152 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:39,433 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:41,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:24:41,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:24:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:24:41,347 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:41,349 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:41,349 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:41,350 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:42,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:43,154 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:44,267 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:44,269 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:44,269 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:44,271 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:45,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:45,156 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:45,271 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:47,186 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:47,189 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:47,190 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:47,191 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:48,158 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:49,159 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:50,105 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:50,107 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:50,108 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:50,109 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:50,160 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:51,109 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:51,160 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:53,025 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:53,027 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:53,027 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:53,029 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:53,162 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:55,163 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:55,945 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:55,947 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:55,947 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:55,949 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:56,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:24:56,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:24:56,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:24:56,164 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:24:56,349 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:24:56,349 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:24:57,164 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:24:58,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:24:58,867 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:24:58,867 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:24:58,868 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:24:59,166 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:01,785 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:01,787 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:01,788 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:01,788 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:01,789 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:02,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:02,168 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:04,169 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:04,710 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:04,712 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:04,712 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:04,714 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:05,170 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:06,171 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:07,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:07,665 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:07,666 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:07,666 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:07,667 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:08,172 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:10,173 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:10,592 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:10,594 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:10,595 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:10,596 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:11,078 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:25:11,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:25:11,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:25:11,174 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:12,175 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:13,333 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:13,611 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:13,613 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:13,613 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:13,614 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:14,176 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:16,177 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:16,539 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:16,541 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:16,542 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:16,543 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:17,178 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:18,179 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:18,544 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:19,489 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:19,490 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:19,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:19,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:20,181 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:22,182 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:22,417 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:22,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:22,420 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:22,421 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:23,183 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:24,183 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:24,422 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:25,346 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:25,348 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:25,349 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:25,350 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:25:26,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:25:26,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:25:26,185 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:26,267 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:25:28,186 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:28,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:28,277 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:28,278 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:28,279 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:29,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:30,187 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:30,317 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:31,205 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:31,207 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:31,207 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:31,209 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:32,189 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:34,128 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:34,130 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:34,130 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:34,131 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:34,191 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:34,191 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:36,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:36,192 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:37,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:37,053 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:37,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:37,055 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:37,193 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:39,974 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:39,976 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:39,976 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:39,977 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:40,195 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:40,195 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:41,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:25:41,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:25:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:25:41,256 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:42,196 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:42,894 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:42,896 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:42,896 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:42,897 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:43,197 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:45,815 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:45,817 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:45,817 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:45,819 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:46,199 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:46,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:46,819 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:48,200 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:48,736 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:48,738 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:48,738 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:48,740 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:49,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:50,202 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:51,657 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:51,659 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:51,659 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:51,660 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:52,204 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:52,661 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:25:54,205 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:54,578 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:54,580 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:54,581 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:54,582 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:55,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:56,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:25:56,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:25:56,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:25:56,206 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:25:56,268 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:25:57,587 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:25:57,589 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:25:57,589 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:25:57,591 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:25:58,208 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:25:58,629 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:00,209 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:00,527 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:00,528 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:00,529 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:00,530 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:01,210 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:02,211 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:03,454 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:03,456 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:03,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:03,458 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:04,212 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:04,458 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:06,213 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:06,397 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:06,399 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:06,399 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:06,401 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:07,214 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:08,215 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:09,345 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:09,347 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:09,347 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:09,349 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:10,216 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:10,349 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:11,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:26:11,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:26:11,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:26:12,217 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:12,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:12,288 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:12,288 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:12,289 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:13,218 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:14,219 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:15,235 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:15,237 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:15,237 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:15,238 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:16,220 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:16,239 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:18,162 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:18,163 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:18,164 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:18,165 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:18,222 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:18,222 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:20,223 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:21,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:21,092 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:21,092 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:21,093 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:21,224 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:22,094 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:24,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:24,018 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:24,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:24,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:24,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:24,226 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:26:26,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:26:26,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:26:26,227 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:26,264 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:26:26,945 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:26,947 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:26,947 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:26,948 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:27,228 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:27,989 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:29,888 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:29,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:29,891 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:29,892 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:30,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:30,230 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:32,231 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:32,843 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:32,845 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:32,846 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:32,847 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:33,232 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:33,847 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:34,233 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:35,767 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:35,769 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:35,770 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:35,771 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:36,234 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:38,235 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:38,816 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:38,819 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:38,819 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:38,820 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:39,236 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:39,821 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:40,237 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:41,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:26:41,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:26:41,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:26:41,765 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:41,767 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:41,767 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:41,768 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:42,238 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:44,239 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:44,702 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:44,705 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:44,705 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:44,706 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:45,240 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:45,707 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:46,241 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:47,648 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:47,650 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:47,650 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:47,651 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:48,242 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:50,243 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:50,568 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:50,570 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:50,570 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:50,572 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:51,244 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:51,572 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:52,245 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:53,489 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:53,491 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:53,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:53,493 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:54,246 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:56,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:26:56,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:26:56,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:26:56,247 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:56,332 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:26:56,430 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:56,432 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:56,432 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:56,433 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:26:57,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:26:57,473 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:26:58,249 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:26:59,374 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:26:59,376 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:26:59,377 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:26:59,378 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:00,251 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:02,252 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:02,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:02,302 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:02,302 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:02,303 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:03,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:03,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:04,253 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:05,230 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:05,232 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:05,232 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:05,233 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:05,254 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:08,150 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:08,153 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:08,153 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:08,154 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:08,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:08,256 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:09,155 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:10,257 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:11,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:27:11,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:27:11,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:27:11,087 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:11,287 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:11,287 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:11,288 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:12,259 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:14,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:14,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:14,012 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:14,013 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:14,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:14,261 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:15,013 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:16,262 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:16,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:16,950 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:16,950 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:16,951 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:17,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:18,263 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:19,867 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:19,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:19,870 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:19,871 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:20,265 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:20,872 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:22,266 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:22,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:22,843 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:22,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:22,845 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:23,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:24,267 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:25,764 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:25,766 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:25,767 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:25,768 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:27:26,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:27:26,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:27:26,269 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:26,298 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:27:26,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:28,270 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:28,687 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:28,688 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:28,689 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:28,690 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:29,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:30,271 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:31,624 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:31,627 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:31,627 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:31,628 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:31,629 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:32,273 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:34,274 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:34,544 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:34,546 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:34,547 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:34,548 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:35,275 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:36,275 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:37,463 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:37,465 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:37,466 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:37,466 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:37,467 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:38,277 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:40,278 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:40,385 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:40,388 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:40,388 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:40,389 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:41,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:27:41,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:27:41,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:27:41,279 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:42,280 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:43,280 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:43,326 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:43,327 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:43,327 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:43,328 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:44,281 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:46,246 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:46,248 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:46,249 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:46,250 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:46,283 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:46,283 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:48,284 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:49,168 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:49,170 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:49,171 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:49,171 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:49,172 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:49,285 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:52,090 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:52,093 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:52,093 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:52,094 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:52,287 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:52,287 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:54,301 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:55,011 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:55,014 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:55,014 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:55,015 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:27:55,016 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:55,302 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:27:56,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:27:56,079 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:27:56,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:27:56,273 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:27:57,939 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:27:57,941 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:27:57,941 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:27:57,942 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:27:58,304 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:27:58,304 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:00,305 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:00,866 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:00,868 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:00,869 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:00,869 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:00,870 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:01,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:02,306 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:03,841 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:03,844 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:03,844 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:03,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:04,308 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:06,309 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:06,761 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:06,765 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:06,765 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:06,765 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:06,766 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:07,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:08,310 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:09,721 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:09,724 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:09,724 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:09,725 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:10,312 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:11,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:28:11,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:28:11,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:28:12,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:12,313 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:12,645 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:12,646 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:12,646 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:12,648 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:13,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:14,314 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:15,580 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:15,583 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:15,583 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:15,584 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:16,316 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:17,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:18,317 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:18,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:18,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:18,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:18,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:19,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:20,318 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:21,429 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:21,431 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:21,432 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:21,433 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:22,320 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:23,433 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:24,321 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:24,356 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:24,359 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:24,359 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:24,360 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:25,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:26,079 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:28:26,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:28:26,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:28:26,322 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:26,326 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:28:27,279 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:27,281 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:27,281 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:27,282 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:27,323 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:29,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:30,209 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:30,211 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:30,211 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:30,212 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:30,325 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:30,325 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:32,326 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:33,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:33,138 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:33,139 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:33,140 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:33,327 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:35,140 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:36,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:36,109 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:36,109 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:36,110 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:36,329 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:36,329 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:38,330 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:39,047 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:39,049 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:39,050 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:39,051 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:39,331 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:40,332 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:41,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:28:41,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:28:41,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:28:41,988 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:41,990 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:41,990 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:41,991 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:42,333 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:44,334 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:44,918 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:44,920 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:44,920 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:44,922 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:45,335 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:46,336 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:46,923 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:48,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:48,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:48,011 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:48,013 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:48,338 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:50,339 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:50,935 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:50,937 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:50,938 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:50,939 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:51,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:51,940 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:52,340 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:53,866 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:53,868 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:53,868 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:53,870 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:54,342 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:56,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:28:56,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:28:56,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:28:56,316 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:28:56,343 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:56,809 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:56,810 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:56,810 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:56,811 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:28:57,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:28:57,853 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:28:58,344 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:28:59,740 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:28:59,742 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:28:59,742 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:28:59,743 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:00,346 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:02,347 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:02,666 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:02,668 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:02,669 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:02,670 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:03,348 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:03,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:04,348 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:05,595 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:05,597 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:05,598 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:05,600 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:06,350 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:08,351 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:08,521 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:08,523 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:08,523 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:08,525 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:09,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:09,525 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:10,352 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:11,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:29:11,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:29:11,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:29:11,447 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:11,449 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:11,449 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:11,450 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:12,354 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:14,355 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:14,374 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:14,377 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:14,377 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:14,378 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:15,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:15,379 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:16,356 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:17,319 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:17,322 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:17,322 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:17,323 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:17,357 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:20,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:20,266 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:20,266 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:20,267 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:20,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:20,359 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:21,268 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:22,360 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:23,184 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:23,186 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:23,187 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:23,188 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:23,361 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:26,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:29:26,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:29:26,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:29:26,137 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:26,342 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:26,343 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:26,343 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:26,344 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:26,344 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:29:26,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:26,363 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:28,364 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:29,063 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:29,066 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:29,066 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:29,067 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:29,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:30,365 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:32,044 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:32,046 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:32,046 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:32,046 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:32,048 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:32,367 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:34,368 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:34,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:34,979 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:34,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:34,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:35,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:36,369 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:37,924 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:37,926 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:37,926 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:37,927 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:37,929 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:38,371 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:40,372 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:40,850 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:40,852 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:40,852 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:40,854 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:29:41,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:29:41,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:29:41,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:42,373 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:43,261 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:43,785 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:43,786 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:43,787 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:43,788 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:44,375 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:46,376 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:46,712 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:46,715 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:46,715 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:46,716 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:47,377 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:48,377 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:48,717 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:49,638 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:49,641 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:49,641 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:49,642 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:50,379 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:52,380 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:52,566 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:52,569 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:52,569 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:52,570 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:53,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:54,381 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:54,571 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:29:55,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:55,490 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:55,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:55,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:56,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:29:56,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:29:56,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:29:56,284 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:29:56,383 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:29:58,384 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:29:58,426 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:29:58,427 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:29:58,427 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:29:58,428 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:29:59,385 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:00,386 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:00,469 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:01,378 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:01,380 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:01,381 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:01,382 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:01,387 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:04,299 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:04,301 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:04,302 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:04,303 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:04,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:04,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:06,303 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:06,389 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:07,234 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:07,237 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:07,237 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:07,238 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:07,390 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:10,163 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:10,165 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:10,165 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:10,166 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:10,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:10,392 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:11,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:30:11,080 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:30:11,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:30:11,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:12,393 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:13,107 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:13,109 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:13,109 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:13,110 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:13,394 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:14,395 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:16,138 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:16,141 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:16,141 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:16,142 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:16,396 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:17,143 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:18,397 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:19,075 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:19,078 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:19,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:19,079 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:19,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:20,399 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:22,015 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:22,017 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:22,018 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:22,019 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:22,401 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:23,019 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:24,402 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:24,942 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:24,944 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:24,945 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:24,946 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:25,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:26,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:30:26,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:30:26,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:30:26,255 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:30:26,403 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:27,868 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:27,870 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:27,870 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:27,871 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:28,405 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:28,909 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:30,406 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:30,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:30,816 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:30,817 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:30,818 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:31,407 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:32,408 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:33,772 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:33,774 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:33,775 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:33,776 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:34,409 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:34,776 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:36,410 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:36,711 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:36,714 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:36,715 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:36,716 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:37,411 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:38,412 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:39,676 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:39,678 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:39,679 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:39,681 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:40,413 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:40,682 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:41,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:30:41,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:30:41,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:30:42,414 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:42,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:42,609 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:42,609 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:42,611 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:43,415 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:44,415 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:45,547 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:45,551 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:45,551 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:45,552 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:46,417 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:46,553 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:48,418 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:48,500 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:48,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:48,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:48,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:49,419 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:50,419 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:51,437 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:51,439 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:51,440 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:51,441 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:52,421 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:52,441 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:30:54,384 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:54,387 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:54,387 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:54,388 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:54,422 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:54,422 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:56,080 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:30:56,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:30:56,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:30:56,258 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:30:56,423 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:30:57,321 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:30:57,323 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:30:57,323 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:30:57,324 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:30:57,424 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:30:58,365 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:00,339 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:00,343 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:00,343 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:00,344 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:00,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:00,426 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:02,427 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:03,276 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:03,278 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:03,279 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:03,280 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:03,428 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:04,280 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:06,204 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:06,207 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:06,207 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:06,208 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:06,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:06,430 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:08,431 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:09,132 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:09,134 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:09,135 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:09,136 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:09,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:10,136 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:10,432 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:11,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:31:11,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:31:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:31:12,103 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:12,105 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:12,105 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:12,107 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:12,434 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:14,435 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:15,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:15,036 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:15,036 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:15,038 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:15,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:16,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:16,436 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:17,981 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:17,983 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:17,983 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:17,985 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:18,438 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:20,439 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:20,908 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:20,910 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:20,910 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:20,912 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:21,440 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:21,912 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:22,440 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:23,833 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:23,835 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:23,836 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:23,837 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:24,442 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:26,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:31:26,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:31:26,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:31:26,311 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:31:26,443 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:26,762 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:26,765 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:26,765 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:26,766 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:27,444 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:27,805 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:28,444 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:29,685 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:29,687 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:29,688 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:29,689 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:30,446 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:32,446 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:32,620 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:32,622 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:32,623 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:32,624 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:33,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:33,624 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:34,448 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:35,572 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:35,576 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:35,576 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:35,577 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:36,450 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:38,451 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:38,499 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:38,501 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:38,502 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:38,503 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:38,926 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:38,926 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:39,452 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:40,452 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:41,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:31:41,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:31:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:31:42,120 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:42,121 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:42,121 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:42,123 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:42,453 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:44,161 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:44,454 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:45,142 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:45,144 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:45,145 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:45,146 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:45,455 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:46,456 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:48,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:48,071 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:48,071 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:48,072 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:48,457 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:50,073 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:50,458 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:50,996 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:50,999 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:50,999 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:51,001 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:51,459 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:52,460 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:53,924 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:53,927 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:53,927 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:53,928 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:54,461 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:55,928 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:31:56,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:31:56,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:31:56,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:31:56,314 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:31:56,463 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:56,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:56,866 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:56,866 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:56,867 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:31:57,464 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:31:58,464 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:31:59,792 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:31:59,794 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:31:59,795 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:31:59,796 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:00,466 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:01,796 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:02,467 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:02,717 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:02,720 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:02,720 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:02,721 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:03,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:04,468 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:05,674 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:05,677 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:05,677 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:05,678 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:06,470 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:07,679 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:08,471 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:08,628 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:08,630 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:08,631 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:08,632 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:09,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:10,472 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:11,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:32:11,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:32:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:32:11,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:11,556 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:11,556 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:11,557 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:12,474 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:13,598 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:14,475 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:14,495 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:14,498 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:14,498 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:14,499 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:15,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:16,476 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:17,420 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:17,422 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:17,422 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:17,424 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:17,477 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:19,424 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:20,347 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:20,350 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:20,350 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:20,351 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:20,479 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:20,479 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:22,480 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:23,283 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:23,285 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:23,286 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:23,287 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:23,481 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:24,482 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:25,287 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:26,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:32:26,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:32:26,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:32:26,270 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:32:26,298 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:26,300 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:26,300 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:26,301 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:26,484 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:28,485 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:29,222 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:29,226 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:29,226 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:29,228 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:29,486 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:30,487 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:31,228 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:32,167 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:32,169 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:32,170 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:32,171 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:32,488 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:34,490 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:35,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:35,095 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:35,095 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:35,097 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:35,491 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:36,492 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:37,097 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:38,014 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:38,016 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:38,016 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:38,017 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:38,493 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:40,494 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:40,933 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:40,936 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:40,936 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:40,937 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:41,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:32:41,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:32:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:32:41,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:42,262 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:42,496 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:43,858 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:43,860 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:43,861 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:43,863 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:44,498 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:46,499 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:46,780 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:46,782 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:46,782 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:46,783 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:47,500 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:47,784 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:48,501 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:49,700 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:49,703 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:49,703 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:49,704 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:50,502 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:52,504 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:52,622 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:52,624 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:52,625 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:52,626 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:53,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:53,626 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:32:54,505 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:55,543 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:55,545 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:55,546 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:55,547 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:56,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:32:56,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:32:56,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:32:56,353 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:32:56,507 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:58,464 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:32:58,466 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:32:58,466 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:32:58,467 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:32:58,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:32:58,509 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:32:59,513 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:00,510 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:01,386 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:01,389 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:01,389 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:01,391 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:01,512 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:04,307 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:04,310 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:04,310 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:04,311 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:04,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:04,514 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:05,311 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:06,515 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:07,228 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:07,231 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:07,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:07,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:07,517 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:08,517 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:10,240 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:10,242 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:10,243 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:10,244 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:10,519 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:11,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:33:11,081 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:33:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:33:11,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:12,520 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:13,161 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:13,163 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:13,163 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:13,164 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:13,521 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:14,522 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:16,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:16,086 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:16,087 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:16,090 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:16,524 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:17,091 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:18,525 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:19,005 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:19,007 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:19,008 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:19,009 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:19,526 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:20,527 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:21,927 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:21,929 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:21,930 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:21,931 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:22,528 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:22,931 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:24,530 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:24,849 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:24,851 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:24,852 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:24,853 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:25,531 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:26,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:33:26,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:33:26,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:33:26,315 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:33:26,532 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:27,772 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:27,773 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:27,774 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:27,775 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:28,534 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:28,813 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:30,535 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:30,693 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:30,695 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:30,695 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:30,696 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:31,536 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:32,537 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:33,613 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:33,616 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:33,616 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:33,617 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:34,538 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:34,618 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:36,535 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:36,537 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:36,537 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:36,539 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:36,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:36,540 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:38,541 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:39,456 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:39,459 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:39,459 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:39,460 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:39,543 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:40,460 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:41,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:33:41,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:33:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:33:42,379 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:42,381 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:42,381 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:42,382 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:42,545 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:42,545 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:44,546 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:45,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:45,302 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:45,303 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:45,304 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:45,547 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:46,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:46,548 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:48,220 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:48,223 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:48,223 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:48,225 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:48,549 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:50,550 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:51,237 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:51,240 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:51,240 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:51,242 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:51,551 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:52,242 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:52,552 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:54,158 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:54,161 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:54,162 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:54,163 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:54,554 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:56,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:33:56,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:33:56,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:33:56,293 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:33:56,555 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:33:57,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:33:57,084 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:33:57,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:33:57,086 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:33:57,556 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:33:58,125 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:33:58,556 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:00,009 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:00,011 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:00,012 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:00,013 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:00,558 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:02,559 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:03,354 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:03,357 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:03,357 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:03,357 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:03,358 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:03,560 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:04,561 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:06,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:06,278 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:06,278 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:06,279 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:06,562 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:08,563 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:09,198 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:09,200 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:09,201 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:09,201 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:09,202 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:09,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:10,564 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:11,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:34:11,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:34:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:34:12,120 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:12,122 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:12,122 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:12,123 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:12,566 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:14,567 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:15,051 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:15,053 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:15,054 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:15,054 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:15,055 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:15,568 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:16,568 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:18,003 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:18,005 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:18,006 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:18,007 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:18,570 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:20,570 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:20,940 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:20,943 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:20,943 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:20,943 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:20,945 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:21,571 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:22,572 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:23,865 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:23,891 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:23,892 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:23,893 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:24,573 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:26,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:34:26,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:34:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:34:26,293 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:34:26,293 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:26,574 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:26,789 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:26,791 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:26,791 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:26,792 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:27,575 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:28,576 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:29,720 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:29,722 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:29,723 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:29,724 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:30,577 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:31,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:32,578 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:32,659 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:32,661 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:32,662 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:32,663 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:33,579 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:34,580 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:35,643 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:35,645 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:35,645 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:35,647 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:36,582 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:37,647 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:38,568 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:38,570 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:38,570 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:38,572 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:38,583 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:38,583 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:40,584 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:41,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:34:41,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:34:41,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:34:41,505 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:41,507 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:41,507 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:41,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:41,585 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:43,554 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:44,446 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:44,448 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:44,448 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:44,450 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:44,587 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:44,588 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:46,589 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:47,372 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:47,374 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:47,374 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:47,375 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:47,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:48,590 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:49,376 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:50,297 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:50,299 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:50,299 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:50,304 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:50,592 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:52,593 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:53,222 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:53,225 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:53,225 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:53,227 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:53,594 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:54,595 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:55,227 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:34:56,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:34:56,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:34:56,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:34:56,160 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:56,261 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:56,261 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:56,262 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:56,263 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:34:56,596 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:34:58,597 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:34:59,087 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:34:59,089 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:34:59,089 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:34:59,091 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:34:59,598 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:00,599 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:01,092 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:02,016 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:02,019 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:02,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:02,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:02,601 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:04,602 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:04,964 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:04,967 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:04,967 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:04,968 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:05,603 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:06,604 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:06,969 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:07,895 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:07,897 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:07,898 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:07,899 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:08,606 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:10,607 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:10,819 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:10,821 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:10,822 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:10,823 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:11,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:35:11,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:35:11,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:35:11,608 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:12,356 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:12,609 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:13,746 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:13,747 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:13,748 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:13,749 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:14,610 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:16,611 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:16,765 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:16,767 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:16,768 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:16,769 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:17,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:17,769 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:18,613 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:19,706 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:19,709 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:19,709 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:19,710 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:20,615 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:22,616 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:22,634 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:22,636 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:22,637 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:22,638 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:23,617 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:23,638 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:24,618 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:25,589 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:25,591 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:25,592 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:25,593 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:25,619 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:26,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:35:26,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:35:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:35:26,357 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:35:28,517 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:28,518 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:28,519 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:28,520 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:28,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:28,621 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:29,561 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:30,622 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:31,447 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:31,449 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:31,449 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:31,450 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:31,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:32,624 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:34,391 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:34,394 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:34,394 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:34,395 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:34,626 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:35,396 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:36,627 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:37,333 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:37,335 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:37,336 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:37,337 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:37,628 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:38,629 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:40,261 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:40,263 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:40,264 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:40,265 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:40,630 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:41,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:35:41,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:35:41,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:35:41,320 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:42,631 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:43,189 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:43,191 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:43,191 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:43,192 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:43,633 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:44,634 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:46,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:46,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:46,120 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:46,121 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:46,635 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:47,121 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:48,636 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:49,045 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:49,047 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:49,047 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:49,048 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:49,637 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:50,638 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:51,972 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:51,974 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:51,975 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:51,977 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:52,640 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:52,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:35:54,641 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:54,918 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:54,920 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:54,921 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:54,922 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:55,642 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:56,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:35:56,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:35:56,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:35:56,360 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:35:56,643 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:35:57,850 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:35:57,852 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:35:57,852 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:35:57,853 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:35:58,644 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:35:58,893 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:00,645 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:00,877 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:00,880 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:00,880 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:00,881 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:01,647 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:02,647 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:03,810 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:03,812 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:03,812 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:03,813 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:04,649 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:04,814 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:06,650 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:06,756 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:06,758 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:06,759 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:06,760 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:07,651 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:08,652 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:09,688 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:09,691 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:09,691 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:09,692 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:10,654 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:10,692 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:11,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:36:11,082 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:36:11,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:36:12,623 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:12,624 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:12,625 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:12,626 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:12,655 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:12,656 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:14,657 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:15,552 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:15,554 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:15,555 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:15,556 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:15,658 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:16,556 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:16,659 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:18,481 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:18,483 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:18,484 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:18,485 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:18,660 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:20,662 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:21,440 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:21,442 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:21,443 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:21,444 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:21,663 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:22,444 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:22,664 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:24,366 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:24,371 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:24,371 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:24,373 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:24,666 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:26,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:36:26,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:36:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:36:26,301 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:36:26,667 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:27,300 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:27,302 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:27,302 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:27,303 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:27,668 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:28,341 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:28,669 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:30,228 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:30,231 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:30,231 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:30,232 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:30,671 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:32,672 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:33,157 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:33,159 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:33,159 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:33,160 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:33,673 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:34,161 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:34,674 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:36,088 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:36,090 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:36,091 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:36,092 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:36,676 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:38,677 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:39,017 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:39,019 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:39,019 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:39,020 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:39,678 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:40,021 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:40,679 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:36:41,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:36:41,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:36:41,944 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:41,946 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:41,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:41,947 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:42,680 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:44,681 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:44,967 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:44,970 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:44,970 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:44,971 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:45,682 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:45,972 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:46,683 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:47,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:47,917 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:47,918 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:47,919 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:48,684 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:50,685 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:50,842 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:50,845 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:50,845 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:50,846 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:51,686 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:51,847 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:52,687 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:53,786 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:53,789 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:53,789 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:53,790 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:54,688 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:56,082 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:36:56,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:36:56,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:36:56,325 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:36:56,689 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:56,714 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:56,716 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:56,716 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:56,717 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:57,690 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:36:57,757 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:36:58,691 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:36:59,642 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:36:59,645 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:36:59,645 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:36:59,646 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:36:59,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:00,692 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:02,604 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:02,606 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:02,607 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:02,608 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:02,694 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:03,608 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:04,695 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:05,532 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:05,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:05,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:05,536 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:05,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:06,696 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:08,459 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:08,462 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:08,462 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:08,463 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:08,698 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:09,464 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:10,699 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:37:11,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:37:11,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:37:11,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:11,389 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:11,389 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:11,390 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:11,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:12,700 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:14,331 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:14,338 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:14,339 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:14,340 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:14,702 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:15,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:16,703 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:17,259 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:17,261 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:17,262 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:17,263 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:17,704 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:18,704 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:20,186 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:20,189 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:20,189 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:20,190 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:20,706 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:21,191 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:22,707 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:23,115 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:23,117 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:23,118 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:23,119 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:23,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:24,708 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:26,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:37:26,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:37:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:37:26,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:26,304 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:26,304 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:26,304 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:26,306 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:26,306 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:37:26,710 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:28,711 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:29,076 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:29,078 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:29,078 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:29,080 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:29,712 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:30,712 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:32,003 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:32,006 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:32,006 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:32,007 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:32,007 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:32,714 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:34,715 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:34,930 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:34,932 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:34,933 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:34,934 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:35,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:36,716 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:37,857 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:37,859 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:37,860 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:37,860 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:37,861 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:38,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:40,718 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:40,806 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:40,808 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:40,809 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:40,810 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:37:41,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:37:41,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:37:41,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:42,720 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:43,327 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:43,740 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:43,742 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:43,742 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:43,743 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:44,722 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:46,668 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:46,670 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:46,670 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:46,671 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:46,723 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:46,723 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:48,672 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:48,724 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:49,605 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:49,607 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:49,607 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:49,608 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:49,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:50,725 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:52,541 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:52,545 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:52,545 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:52,547 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:52,727 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:54,548 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:37:54,728 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:55,482 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:55,484 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:55,485 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:55,486 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:55,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:37:56,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:37:56,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:37:56,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:37:56,334 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:37:56,729 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:37:58,431 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:37:58,433 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:37:58,433 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:37:58,434 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:37:58,731 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:00,473 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:00,732 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:01,359 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:01,361 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:01,361 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:01,363 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:01,733 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:02,733 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:04,287 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:04,290 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:04,290 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:04,291 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:04,735 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:06,292 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:06,736 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:07,214 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:07,216 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:07,216 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:07,218 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:07,737 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:08,738 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:10,243 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:10,246 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:10,246 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:10,247 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:10,739 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:38:11,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:38:11,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:38:12,263 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:12,740 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:13,165 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:13,167 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:13,167 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:13,168 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:13,741 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:14,742 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:16,093 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:16,096 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:16,096 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:16,097 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:16,743 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:18,098 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:18,744 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:19,015 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:19,017 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:19,017 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:19,019 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:19,745 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:20,746 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:21,936 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:21,938 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:21,938 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:21,940 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:22,747 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:23,940 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:24,748 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:25,708 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:25,710 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:25,711 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:25,712 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:25,749 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:26,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:38:26,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:38:26,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:38:26,270 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:38:26,750 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:28,605 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:28,606 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:28,607 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:28,608 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:28,751 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:29,649 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:30,752 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:31,501 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:31,503 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:31,504 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:31,505 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:31,753 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:32,754 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:34,408 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:34,410 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:34,411 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:34,412 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:34,755 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:35,412 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:36,756 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:37,326 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:37,329 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:37,329 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:37,330 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:37,758 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:38,758 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:40,236 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:40,238 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:40,239 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:40,240 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:40,759 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:38:41,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:38:41,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:38:41,363 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:42,760 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:43,139 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:43,141 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:43,141 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:43,142 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:43,761 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:44,762 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:46,064 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:46,066 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:46,067 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:46,068 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:46,764 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:47,068 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:48,764 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:48,987 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:48,989 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:48,989 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:48,991 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:49,766 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:50,766 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:51,890 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:51,893 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:51,893 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:51,894 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:52,768 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:52,895 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:54,769 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:54,794 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:54,840 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:54,840 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:54,841 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:55,770 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:56,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:38:56,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:38:56,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:38:56,251 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:38:56,770 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:38:57,711 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:38:57,713 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:38:57,713 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:38:57,714 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:38:57,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:38:58,753 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:38:58,771 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:00,675 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:00,678 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:00,678 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:00,680 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:00,773 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:02,774 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:03,578 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:03,580 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:03,580 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:03,582 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:03,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:04,582 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:04,775 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:06,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:06,490 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:06,490 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:06,491 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:06,777 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:08,778 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:09,387 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:09,390 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:09,390 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:09,391 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:09,779 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:10,392 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:10,780 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:11,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:39:11,083 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:39:11,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:39:12,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:12,288 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:12,288 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:12,304 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:12,781 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:14,782 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:15,206 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:15,209 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:15,209 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:15,210 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:15,783 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:16,211 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:16,784 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:18,106 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:18,109 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:18,109 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:18,110 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:18,785 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:20,786 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:21,025 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:21,027 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:21,027 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:21,028 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:21,787 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:22,029 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:22,788 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:23,924 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:23,927 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:23,927 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:23,928 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:24,789 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:26,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:39:26,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:39:26,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:39:26,293 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:39:26,790 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:26,824 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:26,825 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:26,826 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:26,827 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:27,791 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:27,869 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:28,792 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:29,740 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:29,743 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:29,743 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:29,744 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:29,793 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:30,793 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:32,681 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:32,683 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:32,684 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:32,685 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:32,795 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:33,686 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:34,795 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:35,583 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:35,585 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:35,586 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:35,587 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:35,796 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:36,797 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:38,488 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:38,491 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:38,491 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:38,492 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:38,798 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:39,493 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:40,799 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:41,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:39:41,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:39:41,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:39:41,530 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:41,532 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:41,532 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:41,533 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:41,800 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:42,801 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:44,439 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:44,447 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:44,447 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:44,448 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:44,802 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:45,449 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:46,804 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:47,361 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:47,363 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:47,363 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:47,364 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:47,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:48,805 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:50,285 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:50,287 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:50,288 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:50,289 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:50,807 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:51,289 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:52,808 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:53,195 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:53,198 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:53,198 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:53,199 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:53,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:54,809 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:56,083 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:39:56,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:39:56,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:39:56,294 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:39:56,294 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:39:56,324 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:56,325 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:56,325 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:56,326 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:56,810 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:39:58,811 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:39:59,249 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:39:59,251 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:39:59,252 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:39:59,253 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:39:59,812 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:00,813 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:02,201 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:02,204 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:02,204 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:02,204 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:02,205 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:02,814 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:04,815 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:05,117 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:05,119 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:05,119 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:05,121 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:05,816 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:06,817 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:08,031 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:08,033 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:08,034 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:08,034 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:08,035 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:08,818 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:10,819 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:10,990 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:10,993 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:10,993 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:10,994 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:11,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:40:11,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:40:11,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:40:11,820 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:12,821 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:13,361 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:13,910 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:13,912 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:13,912 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:13,913 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:14,822 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:16,823 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:16,828 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:16,855 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:16,855 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:16,857 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:17,824 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:18,825 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:18,857 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:19,749 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:19,752 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:19,752 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:19,753 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:19,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:20,826 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:22,669 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:22,671 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:22,672 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:22,673 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:22,828 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:24,674 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:24,829 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:25,725 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:25,728 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:25,728 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:25,729 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:25,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:40:26,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:40:26,085 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:40:26,321 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:40:26,830 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:28,665 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:28,667 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:28,667 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:28,668 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:28,832 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:29,709 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:30,833 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:31,588 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:31,590 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:31,590 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:31,594 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:31,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:32,834 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:34,527 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:34,534 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:34,534 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:34,535 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:34,836 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:35,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:36,837 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:37,440 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:37,442 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:37,442 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:37,443 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:37,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:38,838 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:40,352 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:40,354 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:40,354 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:40,355 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:40,840 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:41,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:40:41,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:40:41,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:40:42,750 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:42,841 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:43,275 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:43,277 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:43,277 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:43,278 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:43,843 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:44,843 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:46,195 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:46,198 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:46,198 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:46,199 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:46,845 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:48,200 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:48,846 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:49,112 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:49,114 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:49,115 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:49,116 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:49,847 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:50,848 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:52,056 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:52,058 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:52,059 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:52,060 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:52,850 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:54,060 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:40:54,851 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:54,986 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:54,988 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:54,988 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:54,989 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:55,852 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:56,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:40:56,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:40:56,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:40:56,359 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:40:56,853 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:57,920 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:40:57,921 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:40:57,921 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:40:57,922 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:40:58,854 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:40:58,855 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:40:59,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:00,857 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:00,860 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:00,861 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:00,862 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:01,857 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:02,858 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:03,806 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:03,809 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:03,809 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:03,810 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:03,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:04,859 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:05,811 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:06,726 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:06,728 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:06,729 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:06,730 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:06,861 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:08,862 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:09,732 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:09,734 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:09,735 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:09,736 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:09,863 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:10,864 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:11,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:41:11,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:41:11,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:41:11,333 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:12,671 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:12,673 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:12,673 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:12,674 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:12,866 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:14,867 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:15,664 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:15,667 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:15,667 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:15,668 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:15,868 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:16,669 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:16,869 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:18,585 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:18,587 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:18,588 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:18,589 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:18,870 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:20,872 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:21,504 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:21,507 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:21,507 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:21,508 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:21,873 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:22,509 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:22,874 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:24,445 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:24,447 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:24,448 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:24,449 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:24,875 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:41:26,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:41:26,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:41:26,282 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:41:26,877 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:27,421 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:27,423 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:27,423 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:27,424 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:27,878 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:28,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:28,069 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:28,879 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:30,880 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:31,194 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:31,197 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:31,197 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:31,198 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:31,881 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:32,882 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:33,199 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:34,116 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:34,118 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:34,118 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:34,119 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:34,884 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:36,885 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:37,038 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:37,041 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:37,041 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:37,042 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:37,886 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:38,887 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:39,042 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:39,959 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:39,962 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:39,962 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:39,963 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:40,888 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:40,888 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:41,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:41:41,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:41:41,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:41:42,882 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:42,884 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:42,884 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:42,885 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:42,890 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:44,891 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:44,925 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:45,807 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:45,809 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:45,810 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:45,811 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:45,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:46,892 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:48,730 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:48,733 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:48,733 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:48,734 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:48,894 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:50,735 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:50,895 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:51,646 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:51,649 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:51,649 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:51,650 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:51,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:52,896 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:54,610 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:54,612 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:54,613 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:54,614 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:54,898 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:56,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:41:56,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:41:56,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:41:56,312 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:41:56,312 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:41:56,899 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:41:57,526 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:41:57,528 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:41:57,528 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:41:57,529 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:41:57,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:41:58,900 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:00,453 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:00,455 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:00,456 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:00,457 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:00,902 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:01,457 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:02,903 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:03,369 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:03,371 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:03,371 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:03,372 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:03,904 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:04,905 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:06,286 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:06,288 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:06,288 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:06,289 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:06,906 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:07,290 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:08,907 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:09,215 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:09,223 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:09,223 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:09,224 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:09,908 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:10,909 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:11,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:42:11,084 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:42:11,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:42:12,133 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:12,135 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:12,136 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:12,137 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:12,910 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:13,181 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:14,911 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:15,050 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:15,052 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:15,052 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:15,054 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:15,912 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:16,913 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:17,977 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:17,979 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:17,980 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:17,981 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:18,914 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:18,915 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:18,981 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:20,915 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:20,917 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:20,917 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:20,918 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:20,919 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:22,920 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:23,834 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:23,836 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:23,836 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:23,838 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:23,921 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:24,838 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:24,921 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:26,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:42:26,085 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:42:26,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:42:26,299 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:42:26,751 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:26,753 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:26,753 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:26,755 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:26,923 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:28,924 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:29,668 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:29,671 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:29,671 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:29,672 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:29,925 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:30,673 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:30,926 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:32,601 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:32,604 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:32,604 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:32,605 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:32,927 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:34,928 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:35,518 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:35,520 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:35,521 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:35,522 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:35,929 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:36,522 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:36,930 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:38,536 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:38,539 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:38,539 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:38,540 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:38,931 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:40,932 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:41,084 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-03 21:42:41,085 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: stop_status +2024-08-03 21:42:41,086 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-03 21:42:41,494 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:41,495 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:41,496 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:41,497 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:41,537 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:41,933 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:42,934 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:44,416 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:44,419 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:44,420 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:44,421 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:44,935 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:46,936 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:47,340 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:47,342 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:47,342 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:47,342 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:47,344 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:47,937 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:48,938 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:50,262 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: partial_history +2024-08-03 21:42:50,264 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:50,264 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:50,266 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:50,939 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:52,917 DEBUG SenderThread:10151 [sender.py:send():382] send: exit +2024-08-03 21:42:52,917 INFO SenderThread:10151 [sender.py:send_exit():589] handling exit code: 255 +2024-08-03 21:42:52,917 INFO SenderThread:10151 [sender.py:send_exit():591] handling runtime: 8336 +2024-08-03 21:42:52,918 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:52,919 INFO SenderThread:10151 [sender.py:send_exit():597] send defer +2024-08-03 21:42:52,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,919 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-03 21:42:52,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:52,919 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,919 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-03 21:42:52,919 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 1 +2024-08-03 21:42:52,919 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,919 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-03 21:42:52,920 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,920 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-03 21:42:52,920 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 2 +2024-08-03 21:42:52,920 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,920 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-03 21:42:52,920 INFO HandlerThread:10151 [system_monitor.py:finish():203] Stopping system monitor +2024-08-03 21:42:52,920 INFO HandlerThread:10151 [interfaces.py:finish():202] Joined cpu monitor +2024-08-03 21:42:52,920 DEBUG SystemMonitor:10151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-03 21:42:52,920 INFO HandlerThread:10151 [interfaces.py:finish():202] Joined disk monitor +2024-08-03 21:42:52,920 DEBUG SystemMonitor:10151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-03 21:42:52,941 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:52,941 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json +2024-08-03 21:42:52,953 INFO HandlerThread:10151 [interfaces.py:finish():202] Joined gpu monitor +2024-08-03 21:42:52,954 INFO HandlerThread:10151 [interfaces.py:finish():202] Joined memory monitor +2024-08-03 21:42:52,954 INFO HandlerThread:10151 [interfaces.py:finish():202] Joined network monitor +2024-08-03 21:42:52,954 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,954 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-03 21:42:52,954 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 3 +2024-08-03 21:42:52,954 DEBUG SenderThread:10151 [sender.py:send():382] send: stats +2024-08-03 21:42:52,954 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,955 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-03 21:42:52,956 DEBUG SenderThread:10151 [sender.py:send():382] send: history +2024-08-03 21:42:52,956 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: summary_record +2024-08-03 21:42:52,957 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:52,958 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,958 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-03 21:42:52,958 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 4 +2024-08-03 21:42:52,958 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,958 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-03 21:42:52,958 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,958 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-03 21:42:52,958 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 5 +2024-08-03 21:42:52,958 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,958 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-03 21:42:52,959 DEBUG SenderThread:10151 [sender.py:send():382] send: summary +2024-08-03 21:42:52,960 INFO SenderThread:10151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-03 21:42:52,960 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,960 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-03 21:42:52,960 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 6 +2024-08-03 21:42:52,960 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,960 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-03 21:42:52,960 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,960 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-03 21:42:52,961 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 7 +2024-08-03 21:42:52,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:42:52,961 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:52,961 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-03 21:42:52,961 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:52,961 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-03 21:42:53,917 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 21:42:54,950 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 8 +2024-08-03 21:42:54,951 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 21:42:54,951 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:54,951 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-03 21:42:54,951 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:54,951 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-03 21:42:54,951 INFO SenderThread:10151 [job_builder.py:build():296] Attempting to build job artifact +2024-08-03 21:42:54,953 INFO SenderThread:10151 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-03 21:42:55,048 INFO SenderThread:10151 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-03 21:42:55,081 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 9 +2024-08-03 21:42:55,081 DEBUG SenderThread:10151 [sender.py:send():382] send: artifact +2024-08-03 21:42:55,081 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:55,083 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-03 21:42:55,918 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-03 21:42:55,943 INFO Thread-12 :10151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log +2024-08-03 21:42:55,964 INFO SenderThread:10151 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA4NTUyMzkyNA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MTk4ODAyMA==', 'versionIndex': 2}}} +2024-08-03 21:42:55,964 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:55,964 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-03 21:42:55,965 INFO SenderThread:10151 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-03 21:42:56,944 INFO SenderThread:10151 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240803_192355-n3hnzq4n/files +2024-08-03 21:42:56,944 INFO SenderThread:10151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_192355-n3hnzq4n/files/requirements.txt requirements.txt +2024-08-03 21:42:56,945 INFO SenderThread:10151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_192355-n3hnzq4n/files/config.yaml config.yaml +2024-08-03 21:42:56,945 INFO SenderThread:10151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-metadata.json wandb-metadata.json +2024-08-03 21:42:56,945 INFO SenderThread:10151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_192355-n3hnzq4n/files/wandb-summary.json wandb-summary.json +2024-08-03 21:42:56,945 INFO SenderThread:10151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240803_192355-n3hnzq4n/files/output.log output.log +2024-08-03 21:42:56,945 INFO SenderThread:10151 [sender.py:transition_state():617] send defer: 10 +2024-08-03 21:42:56,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: poll_exit +2024-08-03 21:42:56,946 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: defer +2024-08-03 21:42:56,946 INFO HandlerThread:10151 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-03 21:42:56,946 DEBUG SenderThread:10151 [sender.py:send_request():409] send_request: defer +2024-08-03 21:42:56,946 INFO SenderThread:10151 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-03 21:42:56,946 INFO SenderThread:10151 [file_pusher.py:finish():172] shutting down file pusher +2024-08-03 21:43:02,947 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:43:07,948 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:43:12,948 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:43:17,949 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:43:22,950 DEBUG HandlerThread:10151 [handler.py:handle_request():146] handle_request: status_report +2024-08-03 21:43:24,873 WARNING StreamThr :10151 [internal.py:is_dead():414] Internal process exiting, parent pid 10080 disappeared +2024-08-03 21:43:24,873 ERROR StreamThr :10151 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-03 21:43:24,950 INFO SenderThread:10151 [sender.py:finish():1572] shutting down sender +2024-08-03 21:43:24,950 INFO SenderThread:10151 [file_pusher.py:finish():172] shutting down file pusher +2024-08-03 21:43:24,950 INFO SenderThread:10151 [file_pusher.py:join():178] waiting for file pusher +2024-08-03 21:43:24,950 INFO HandlerThread:10151 [handler.py:finish():869] shutting down handler +2024-08-03 21:43:24,950 INFO WriterThread:10151 [datastore.py:close():296] close: /project/wandb/run-20240803_192355-n3hnzq4n/run-n3hnzq4n.wandb +2024-08-03 21:43:24,950 INFO SenderThread:10151 [file_stream.py:finish():595] file stream finish called +2024-08-03 21:43:25,282 INFO SenderThread:10151 [file_stream.py:finish():599] file stream finish is done diff --git a/wandb/run-20240803_192355-n3hnzq4n/logs/debug.log b/wandb/run-20240803_192355-n3hnzq4n/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ced6083900fc8a2d4ccc645c152bf2cba3353c38 --- /dev/null +++ b/wandb/run-20240803_192355-n3hnzq4n/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-03 19:23:55,363 INFO MainThread:10080 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_setup.py:_flush():76] Configure stats pid to 10080 +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240803_192355-n3hnzq4n/logs/debug.log +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240803_192355-n3hnzq4n/logs/debug-internal.log +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_init.py:init():566] calling init triggers +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample2_train_2024-08-03-19:23:42', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample2', 'save': '/work/llm_recipes/models/tiny-mistral-sample2', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 1600, 'micro_batch_size': 40, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40} +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_init.py:init():616] starting backend +2024-08-03 19:23:55,364 INFO MainThread:10080 [wandb_init.py:init():620] setting up manager +2024-08-03 19:23:55,369 INFO MainThread:10080 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-03 19:23:55,371 INFO MainThread:10080 [wandb_init.py:init():628] backend started and connected +2024-08-03 19:23:55,375 INFO MainThread:10080 [wandb_init.py:init():720] updated telemetry +2024-08-03 19:23:55,403 INFO MainThread:10080 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-03 19:23:55,888 INFO MainThread:10080 [wandb_run.py:_on_init():2262] communicating current version +2024-08-03 19:23:55,974 INFO MainThread:10080 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-03 19:23:55,974 INFO MainThread:10080 [wandb_init.py:init():804] starting run threads in backend +2024-08-03 19:23:56,034 INFO MainThread:10080 [wandb_run.py:_console_start():2241] atexit reg +2024-08-03 19:23:56,034 INFO MainThread:10080 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-03 19:23:56,034 INFO MainThread:10080 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-03 19:23:56,035 INFO MainThread:10080 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-03 19:23:56,035 INFO MainThread:10080 [wandb_init.py:init():847] run started, returning control to user process +2024-08-03 19:23:57,141 INFO MainThread:10080 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'} +2024-08-03 19:23:57,141 INFO MainThread:10080 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240812_063027-j1htzx7q/files/config.yaml b/wandb/run-20240812_063027-j1htzx7q/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b86be58d812a089c59cb0cc6a0d70488601ff24b --- /dev/null +++ b/wandb/run-20240812_063027-j1htzx7q/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '235289369' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/google/gemma-2-2b +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-sample-gemma-2-2b_train_2024-08-12-06:30:12 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-sample-gemma-2-2b +save: + desc: null + value: /work/llm_recipes/models/yans-sample-gemma-2-2b +base_model: + desc: null + value: /share/pretrained_lm/google/gemma-2-2b +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 3 +save_interval: + desc: null + value: 3 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-sample-gemma-2-2b +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 256000 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723411827.601845 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Gemma2ForCausalLM +activation_function: + desc: null + value: gelu_pytorch_tanh +hidden_size: + desc: null + value: 2304 +model_type: + desc: null + value: gemma2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 8 +num_hidden_layers: + desc: null + value: 26 diff --git a/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt b/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json b/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..df8605f95fa7a367a740a74d3c6433a1056383c1 --- /dev/null +++ b/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-11T21:30:28.265073", + "startedAt": "2024-08-11T21:30:27.589443", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/google/gemma-2-2b", + "--train-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--test-data-path", + "235289369", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "3", + "--eval-interval", + "3", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/google/gemma-2-2b", + "--save", + "/work/llm_recipes/models/yans-sample-gemma-2-2b", + "--load", + "/work/llm_recipes/models/yans-sample-gemma-2-2b", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-sample-gemma-2-2b", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-sample-gemma-2-2b_train_2024-08-12-06:30:12" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0429999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487823486328125 + } +} diff --git a/wandb/run-20240812_063027-j1htzx7q/logs/debug-internal.log b/wandb/run-20240812_063027-j1htzx7q/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..92f09d91701e58a04ec38ecc8e1cf055ac81159a --- /dev/null +++ b/wandb/run-20240812_063027-j1htzx7q/logs/debug-internal.log @@ -0,0 +1,261 @@ +2024-08-12 06:30:27,603 INFO StreamThr :12721 [internal.py:wandb_internal():86] W&B internal server running at pid: 12721, started at: 2024-08-12 06:30:27.602612 +2024-08-12 06:30:27,605 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status +2024-08-12 06:30:27,607 INFO WriterThread:12721 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb +2024-08-12 06:30:27,608 DEBUG SenderThread:12721 [sender.py:send():382] send: header +2024-08-12 06:30:27,640 DEBUG SenderThread:12721 [sender.py:send():382] send: run +2024-08-12 06:30:28,148 INFO SenderThread:12721 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_063027-j1htzx7q/files +2024-08-12 06:30:28,148 INFO SenderThread:12721 [sender.py:_start_run_threads():1136] run started: j1htzx7q with start time 1723411827.601845 +2024-08-12 06:30:28,154 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: check_version +2024-08-12 06:30:28,154 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: check_version +2024-08-12 06:30:28,244 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: run_start +2024-08-12 06:30:28,251 DEBUG HandlerThread:12721 [system_info.py:__init__():27] System info init +2024-08-12 06:30:28,251 DEBUG HandlerThread:12721 [system_info.py:__init__():42] System info init done +2024-08-12 06:30:28,251 INFO HandlerThread:12721 [system_monitor.py:start():194] Starting system monitor +2024-08-12 06:30:28,251 INFO SystemMonitor:12721 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-12 06:30:28,251 INFO HandlerThread:12721 [system_monitor.py:probe():214] Collecting system info +2024-08-12 06:30:28,252 INFO SystemMonitor:12721 [interfaces.py:start():190] Started cpu monitoring +2024-08-12 06:30:28,252 INFO SystemMonitor:12721 [interfaces.py:start():190] Started disk monitoring +2024-08-12 06:30:28,253 INFO SystemMonitor:12721 [interfaces.py:start():190] Started gpu monitoring +2024-08-12 06:30:28,254 INFO SystemMonitor:12721 [interfaces.py:start():190] Started memory monitoring +2024-08-12 06:30:28,255 INFO SystemMonitor:12721 [interfaces.py:start():190] Started network monitoring +2024-08-12 06:30:28,264 DEBUG HandlerThread:12721 [system_info.py:probe():151] Probing system +2024-08-12 06:30:28,267 DEBUG HandlerThread:12721 [system_info.py:_probe_git():136] Probing git +2024-08-12 06:30:28,279 DEBUG HandlerThread:12721 [system_info.py:_probe_git():144] Probing git done +2024-08-12 06:30:28,279 DEBUG HandlerThread:12721 [system_info.py:probe():199] Probing system done +2024-08-12 06:30:28,279 DEBUG HandlerThread:12721 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T21:30:28.265073', 'startedAt': '2024-08-11T21:30:27.589443', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/google/gemma-2-2b', '--train-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--valid-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--test-data-path', '235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '3', '--eval-interval', '3', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/google/gemma-2-2b', '--save', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--load', '/work/llm_recipes/models/yans-sample-gemma-2-2b', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-sample-gemma-2-2b', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-sample-gemma-2-2b_train_2024-08-12-06:30:12'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}} +2024-08-12 06:30:28,279 INFO HandlerThread:12721 [system_monitor.py:probe():224] Finished collecting system info +2024-08-12 06:30:28,279 INFO HandlerThread:12721 [system_monitor.py:probe():227] Publishing system info +2024-08-12 06:30:28,281 INFO HandlerThread:12721 [system_monitor.py:probe():229] Finished publishing system info +2024-08-12 06:30:28,287 DEBUG SenderThread:12721 [sender.py:send():382] send: files +2024-08-12 06:30:28,287 INFO SenderThread:12721 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-12 06:30:28,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: python_packages +2024-08-12 06:30:28,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: python_packages +2024-08-12 06:30:28,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:30:28,298 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:30:28,299 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:30:28,566 DEBUG SenderThread:12721 [sender.py:send():382] send: telemetry +2024-08-12 06:30:28,941 INFO wandb-upload_0:12721 [upload_job.py:push():131] Uploaded file /tmp/tmpagb8lhaywandb/h9wwuria-wandb-metadata.json +2024-08-12 06:30:29,150 INFO Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt +2024-08-12 06:30:29,150 INFO Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:30:29,150 INFO Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json +2024-08-12 06:30:31,151 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:30:32,987 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:30:37,988 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:30:42,988 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:30:43,296 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:30:43,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:30:43,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:30:48,530 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:30:53,531 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:30:58,296 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:30:58,296 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:30:58,340 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:30:58,553 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:30:59,169 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml +2024-08-12 06:31:03,753 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:08,754 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:13,296 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:31:13,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:31:13,336 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:31:14,505 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:19,506 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:24,507 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:28,255 DEBUG SystemMonitor:12721 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-12 06:31:28,257 DEBUG SenderThread:12721 [sender.py:send():382] send: stats +2024-08-12 06:31:28,296 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:31:28,296 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:31:28,340 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:31:30,499 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:35,500 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:40,561 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:41,196 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:31:43,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:31:43,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:31:43,298 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:31:46,508 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:51,509 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:56,510 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:31:58,258 DEBUG SenderThread:12721 [sender.py:send():382] send: stats +2024-08-12 06:31:58,296 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:31:58,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:31:58,340 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:32:02,497 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:07,497 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:12,498 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:13,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:32:13,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:32:13,340 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:32:17,531 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:22,531 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:27,532 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:28,259 DEBUG SenderThread:12721 [sender.py:send():382] send: stats +2024-08-12 06:32:28,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:32:28,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:32:28,340 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:32:33,512 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:38,513 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:43,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:32:43,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:32:43,340 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:32:43,534 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:48,534 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:53,535 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:32:58,260 DEBUG SenderThread:12721 [sender.py:send():382] send: stats +2024-08-12 06:32:58,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:32:58,297 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:32:58,344 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:32:59,520 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:33:01,246 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:33:05,103 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:33:10,126 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:33:11,252 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:33:12,515 DEBUG SenderThread:12721 [sender.py:send():382] send: config +2024-08-12 06:33:12,515 DEBUG SenderThread:12721 [sender.py:send():382] send: config +2024-08-12 06:33:13,253 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:33:13,297 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 06:33:13,298 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: stop_status +2024-08-12 06:33:13,298 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:33:15,255 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:33:15,581 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:33:15,887 DEBUG SenderThread:12721 [sender.py:send():382] send: exit +2024-08-12 06:33:15,887 INFO SenderThread:12721 [sender.py:send_exit():589] handling exit code: 1 +2024-08-12 06:33:15,887 INFO SenderThread:12721 [sender.py:send_exit():591] handling runtime: 167 +2024-08-12 06:33:15,889 INFO SenderThread:12721 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:33:15,889 INFO SenderThread:12721 [sender.py:send_exit():597] send defer +2024-08-12 06:33:15,889 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,889 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-12 06:33:15,889 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,889 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-12 06:33:15,889 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 1 +2024-08-12 06:33:15,890 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,890 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-12 06:33:15,890 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,890 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-12 06:33:15,890 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 2 +2024-08-12 06:33:15,890 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,890 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-12 06:33:15,890 INFO HandlerThread:12721 [system_monitor.py:finish():203] Stopping system monitor +2024-08-12 06:33:15,890 INFO HandlerThread:12721 [interfaces.py:finish():202] Joined cpu monitor +2024-08-12 06:33:15,890 DEBUG SystemMonitor:12721 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-12 06:33:15,891 INFO HandlerThread:12721 [interfaces.py:finish():202] Joined disk monitor +2024-08-12 06:33:15,891 DEBUG SystemMonitor:12721 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-12 06:33:15,924 INFO HandlerThread:12721 [interfaces.py:finish():202] Joined gpu monitor +2024-08-12 06:33:15,925 INFO HandlerThread:12721 [interfaces.py:finish():202] Joined memory monitor +2024-08-12 06:33:15,925 INFO HandlerThread:12721 [interfaces.py:finish():202] Joined network monitor +2024-08-12 06:33:15,925 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,925 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-12 06:33:15,925 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 3 +2024-08-12 06:33:15,926 DEBUG SenderThread:12721 [sender.py:send():382] send: stats +2024-08-12 06:33:15,926 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,926 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-12 06:33:15,926 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,926 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-12 06:33:15,926 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 4 +2024-08-12 06:33:15,926 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,926 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-12 06:33:15,926 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,926 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-12 06:33:15,927 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 5 +2024-08-12 06:33:15,927 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,927 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-12 06:33:15,927 DEBUG SenderThread:12721 [sender.py:send():382] send: summary +2024-08-12 06:33:15,928 INFO SenderThread:12721 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 06:33:15,928 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,928 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-12 06:33:15,928 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 6 +2024-08-12 06:33:15,928 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:15,928 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-12 06:33:15,928 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:15,928 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-12 06:33:15,931 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 06:33:16,132 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 7 +2024-08-12 06:33:16,132 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:16,132 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-12 06:33:16,132 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:16,132 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-12 06:33:16,256 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml +2024-08-12 06:33:16,256 INFO Thread-12 :12721 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-summary.json +2024-08-12 06:33:16,887 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:33:17,100 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 8 +2024-08-12 06:33:17,100 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:33:17,100 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:17,100 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-12 06:33:17,100 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:17,100 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-12 06:33:17,100 INFO SenderThread:12721 [job_builder.py:build():296] Attempting to build job artifact +2024-08-12 06:33:17,101 INFO SenderThread:12721 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-12 06:33:17,116 INFO SenderThread:12721 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-12 06:33:17,124 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 9 +2024-08-12 06:33:17,125 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:17,125 DEBUG SenderThread:12721 [sender.py:send():382] send: artifact +2024-08-12 06:33:17,125 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-12 06:33:17,257 INFO Thread-12 :12721 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:33:17,887 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:33:18,153 INFO SenderThread:12721 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE0MDAxODM0Nw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}} +2024-08-12 06:33:18,154 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:18,154 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-12 06:33:18,154 INFO SenderThread:12721 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-12 06:33:18,258 INFO SenderThread:12721 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_063027-j1htzx7q/files +2024-08-12 06:33:18,258 INFO SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt requirements.txt +2024-08-12 06:33:18,258 INFO SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml config.yaml +2024-08-12 06:33:18,259 INFO SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-metadata.json wandb-metadata.json +2024-08-12 06:33:18,260 INFO SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-summary.json wandb-summary.json +2024-08-12 06:33:18,262 INFO SenderThread:12721 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_063027-j1htzx7q/files/output.log output.log +2024-08-12 06:33:18,262 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 10 +2024-08-12 06:33:18,262 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:33:18,264 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:18,265 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-12 06:33:18,266 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:18,266 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-12 06:33:18,266 INFO SenderThread:12721 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 06:33:18,655 INFO wandb-upload_0:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/requirements.txt +2024-08-12 06:33:18,745 INFO wandb-upload_1:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/config.yaml +2024-08-12 06:33:18,843 INFO wandb-upload_2:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/wandb-summary.json +2024-08-12 06:33:18,858 INFO wandb-upload_3:12721 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_063027-j1htzx7q/files/output.log +2024-08-12 06:33:18,888 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:33:18,889 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:33:19,058 INFO Thread-11 (_thread_body):12721 [sender.py:transition_state():617] send defer: 11 +2024-08-12 06:33:19,059 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:19,059 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-12 06:33:19,059 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:19,059 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-12 06:33:19,059 INFO SenderThread:12721 [file_pusher.py:join():178] waiting for file pusher +2024-08-12 06:33:19,059 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 12 +2024-08-12 06:33:19,059 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:19,059 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-12 06:33:19,059 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:19,059 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-12 06:33:19,059 INFO SenderThread:12721 [file_stream.py:finish():595] file stream finish called +2024-08-12 06:33:19,821 INFO SenderThread:12721 [file_stream.py:finish():599] file stream finish is done +2024-08-12 06:33:19,821 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 13 +2024-08-12 06:33:19,822 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:19,822 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-12 06:33:19,822 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:19,822 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-12 06:33:19,822 INFO SenderThread:12721 [sender.py:transition_state():617] send defer: 14 +2024-08-12 06:33:19,822 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: defer +2024-08-12 06:33:19,823 DEBUG SenderThread:12721 [sender.py:send():382] send: final +2024-08-12 06:33:19,823 INFO HandlerThread:12721 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-12 06:33:19,823 DEBUG SenderThread:12721 [sender.py:send():382] send: footer +2024-08-12 06:33:19,823 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: defer +2024-08-12 06:33:19,823 INFO SenderThread:12721 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-12 06:33:19,823 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:33:19,823 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:33:19,824 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 06:33:19,824 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 06:33:19,824 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: server_info +2024-08-12 06:33:19,824 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: get_summary +2024-08-12 06:33:19,825 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: server_info +2024-08-12 06:33:19,826 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-12 06:33:19,826 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 06:33:19,827 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: job_info +2024-08-12 06:33:19,994 DEBUG SenderThread:12721 [sender.py:send_request():409] send_request: job_info +2024-08-12 06:33:19,994 INFO MainThread:12721 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-12 06:33:19,994 INFO MainThread:12721 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-12 06:33:19,995 INFO MainThread:12721 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-12 06:33:19,995 DEBUG HandlerThread:12721 [handler.py:handle_request():146] handle_request: shutdown +2024-08-12 06:33:19,995 INFO HandlerThread:12721 [handler.py:finish():869] shutting down handler +2024-08-12 06:33:20,827 INFO WriterThread:12721 [datastore.py:close():296] close: /project/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb +2024-08-12 06:33:20,994 INFO SenderThread:12721 [sender.py:finish():1572] shutting down sender +2024-08-12 06:33:20,995 INFO SenderThread:12721 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 06:33:20,995 INFO SenderThread:12721 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240812_063027-j1htzx7q/logs/debug.log b/wandb/run-20240812_063027-j1htzx7q/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a2b7e9745d0459938627c362211c28fe024e78d3 --- /dev/null +++ b/wandb/run-20240812_063027-j1htzx7q/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Configure stats pid to 12650 +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_063027-j1htzx7q/logs/debug.log +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_063027-j1htzx7q/logs/debug-internal.log +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_init.py:init():566] calling init triggers +2024-08-12 06:30:27,595 INFO MainThread:12650 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'test_data_path': ['235289369', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v1_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/google/gemma-2-2b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-sample-gemma-2-2b_train_2024-08-12-06:30:12', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'save': '/work/llm_recipes/models/yans-sample-gemma-2-2b', 'base_model': '/share/pretrained_lm/google/gemma-2-2b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 3, 'save_interval': 3, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-sample-gemma-2-2b', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 256000, 'gradient_accumulation_steps': 320} +2024-08-12 06:30:27,596 INFO MainThread:12650 [wandb_init.py:init():616] starting backend +2024-08-12 06:30:27,596 INFO MainThread:12650 [wandb_init.py:init():620] setting up manager +2024-08-12 06:30:27,600 INFO MainThread:12650 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-12 06:30:27,601 INFO MainThread:12650 [wandb_init.py:init():628] backend started and connected +2024-08-12 06:30:27,606 INFO MainThread:12650 [wandb_init.py:init():720] updated telemetry +2024-08-12 06:30:27,632 INFO MainThread:12650 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-12 06:30:28,153 INFO MainThread:12650 [wandb_run.py:_on_init():2262] communicating current version +2024-08-12 06:30:28,237 INFO MainThread:12650 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-12 06:30:28,237 INFO MainThread:12650 [wandb_init.py:init():804] starting run threads in backend +2024-08-12 06:30:28,296 INFO MainThread:12650 [wandb_run.py:_console_start():2241] atexit reg +2024-08-12 06:30:28,296 INFO MainThread:12650 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-12 06:30:28,296 INFO MainThread:12650 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-12 06:30:28,296 INFO MainThread:12650 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-12 06:30:28,298 INFO MainThread:12650 [wandb_init.py:init():847] run started, returning control to user process +2024-08-12 06:33:12,514 INFO MainThread:12650 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Gemma2ForCausalLM', 'activation_function': 'gelu_pytorch_tanh', 'hidden_size': 2304, 'model_type': 'gemma2', 'max_position_embeddings': 4096, 'num_attention_heads': 8, 'num_hidden_layers': 26} +2024-08-12 06:33:12,515 INFO MainThread:12650 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-12 06:33:20,996 WARNING MsgRouterThr:12650 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb b/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9da3212b02e2a4b2a9930d6bb7033e089c166a88 Binary files /dev/null and b/wandb/run-20240812_063027-j1htzx7q/run-j1htzx7q.wandb differ diff --git a/wandb/run-20240823_163849-faey1t8u/files/config.yaml b/wandb/run-20240823_163849-faey1t8u/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e33fd7655f8fc3c6e1ff5fd8831920cf7b982fa --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +test_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: Qwen2-0.5b-0.2_train_2024-08-23-16:38:35 +wandb_project: + desc: null + value: llm_tutorial-0.2 +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +save: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 7500 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 7500 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 3 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/Qwen2-0.5b-0.2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 106 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724398729.364923 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240823_163849-faey1t8u/files/output.log b/wandb/run-20240823_163849-faey1t8u/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f376f0bb70826b18afe24a21a9f9994b53ab9cc5 --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/files/output.log @@ -0,0 +1,126 @@ +Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 2400000 + validation: 2403200 + test: 3200 +> building train, validation, and test datasets for GPT ... +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +[rank0]:[2024-08-23 16:38:58,062] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000040/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 89.20992749379542, Tokens per sec: 22186.06078570857, Loss: 4.376823425292969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 90.73548605717187, Tokens per sec: 22565.459536162463, Loss: 4.388589382171631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 90.773373714071, Tokens per sec: 22574.882006089374, Loss: 4.334207057952881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 90.78534040748795, Tokens per sec: 22577.85806262267, Loss: 4.347831726074219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 90.91842135677658, Tokens per sec: 22610.954626125003, Loss: 4.369765281677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 90.73681564901436, Tokens per sec: 22565.79019897395, Loss: 4.371013164520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 90.85118295674725, Tokens per sec: 22594.23277383689, Loss: 4.347028732299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 90.81419242163798, Tokens per sec: 22585.033413592068, Loss: 4.319859504699707 +------------------------------------------------------------------ +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train + loss.backward() + File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward + torch.autograd.backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward + _engine_run_backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20240823_163849-faey1t8u/files/requirements.txt b/wandb/run-20240823_163849-faey1t8u/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json b/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..72e98996f983bebc6d3c06b267a1afa02e650ca9 --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-23T07:38:49.894670", + "startedAt": "2024-08-23T07:38:49.352718", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "3", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "7500", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--test-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "7500", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--load", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/Qwen2-0.5b-0.2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial-0.2", + "--wandb-name", + "Qwen2-0.5b-0.2_train_2024-08-23-16:38:35" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487831115722656 + } +} diff --git a/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json b/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7aac0871f3f48c1d20b4412af2496fb66d3d36ea --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 4.319859504699707, "training/perplexity": 75.17806538514934, "utils/batch_size": 3, "utils/global_batch_size": 318, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 106, "utils/iteration": 48, "optimizer/lr": 2.8240000000000004e-06, "optimizer/variance_l2": 0.050272112510912605, "optimizer/variance_sqrt_l2": 0.9579955556165142, "optimizer/momentum_l2": 0.9571913293356521, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.9176788330078125, "optimizer/variance_sqrt_l1": 4099.5, "optimizer/momentum_l1": 3579.0, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.0419921875, "optimizer/variance_sqrt_abs_max": 0.205078125, "optimizer/momentum_abs_max": 0.2236328125, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 57.68625514700034, "stats/tokens_per_sec": 22585.033413592068, "stats/tokens_per_sec_per_gpu": 22585.033413592068, "stats/tflops": 90.81419242163798, "_timestamp": 1724399202.2699971, "_runtime": 472.90507411956787, "_step": 48, "_wandb": {"runtime": 476}} \ No newline at end of file diff --git a/wandb/run-20240823_163849-faey1t8u/logs/debug-internal.log b/wandb/run-20240823_163849-faey1t8u/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..18753d3d599607bb7f7994ac2c4f39ace3a99c5b --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/logs/debug-internal.log @@ -0,0 +1,439 @@ +2024-08-23 16:38:49,367 INFO StreamThr :12305 [internal.py:wandb_internal():86] W&B internal server running at pid: 12305, started at: 2024-08-23 16:38:49.366185 +2024-08-23 16:38:49,368 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status +2024-08-23 16:38:49,370 INFO WriterThread:12305 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb +2024-08-23 16:38:49,371 DEBUG SenderThread:12305 [sender.py:send():382] send: header +2024-08-23 16:38:49,405 DEBUG SenderThread:12305 [sender.py:send():382] send: run +2024-08-23 16:38:49,800 INFO SenderThread:12305 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_163849-faey1t8u/files +2024-08-23 16:38:49,800 INFO SenderThread:12305 [sender.py:_start_run_threads():1136] run started: faey1t8u with start time 1724398729.364923 +2024-08-23 16:38:49,806 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: check_version +2024-08-23 16:38:49,806 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: check_version +2024-08-23 16:38:49,876 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: run_start +2024-08-23 16:38:49,882 DEBUG HandlerThread:12305 [system_info.py:__init__():27] System info init +2024-08-23 16:38:49,882 DEBUG HandlerThread:12305 [system_info.py:__init__():42] System info init done +2024-08-23 16:38:49,882 INFO HandlerThread:12305 [system_monitor.py:start():194] Starting system monitor +2024-08-23 16:38:49,882 INFO SystemMonitor:12305 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-23 16:38:49,882 INFO HandlerThread:12305 [system_monitor.py:probe():214] Collecting system info +2024-08-23 16:38:49,883 INFO SystemMonitor:12305 [interfaces.py:start():190] Started cpu monitoring +2024-08-23 16:38:49,883 INFO SystemMonitor:12305 [interfaces.py:start():190] Started disk monitoring +2024-08-23 16:38:49,884 INFO SystemMonitor:12305 [interfaces.py:start():190] Started gpu monitoring +2024-08-23 16:38:49,885 INFO SystemMonitor:12305 [interfaces.py:start():190] Started memory monitoring +2024-08-23 16:38:49,886 INFO SystemMonitor:12305 [interfaces.py:start():190] Started network monitoring +2024-08-23 16:38:49,894 DEBUG HandlerThread:12305 [system_info.py:probe():151] Probing system +2024-08-23 16:38:49,896 DEBUG HandlerThread:12305 [system_info.py:_probe_git():136] Probing git +2024-08-23 16:38:49,908 DEBUG HandlerThread:12305 [system_info.py:_probe_git():144] Probing git done +2024-08-23 16:38:49,909 DEBUG HandlerThread:12305 [system_info.py:probe():199] Probing system done +2024-08-23 16:38:49,909 DEBUG HandlerThread:12305 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T07:38:49.894670', 'startedAt': '2024-08-23T07:38:49.352718', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '131072', '--micro-batch-size', '3', '--valid_micro_batch_size', '1', '--global-batch-size', '320', '--train-iters', '7500', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '7500', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-16:38:35'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487831115722656}} +2024-08-23 16:38:49,909 INFO HandlerThread:12305 [system_monitor.py:probe():224] Finished collecting system info +2024-08-23 16:38:49,909 INFO HandlerThread:12305 [system_monitor.py:probe():227] Publishing system info +2024-08-23 16:38:49,910 INFO HandlerThread:12305 [system_monitor.py:probe():229] Finished publishing system info +2024-08-23 16:38:49,915 DEBUG SenderThread:12305 [sender.py:send():382] send: files +2024-08-23 16:38:49,916 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-23 16:38:49,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: python_packages +2024-08-23 16:38:49,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:38:49,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:38:49,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: python_packages +2024-08-23 16:38:49,930 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:38:50,183 DEBUG SenderThread:12305 [sender.py:send():382] send: telemetry +2024-08-23 16:38:50,520 INFO wandb-upload_0:12305 [upload_job.py:push():131] Uploaded file /tmp/tmpljn_2vd6wandb/tfd8n6zw-wandb-metadata.json +2024-08-23 16:38:50,802 INFO Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json +2024-08-23 16:38:50,802 INFO Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/requirements.txt +2024-08-23 16:38:50,803 INFO Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:38:52,803 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:38:54,804 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:38:54,958 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:38:56,805 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:38:58,352 DEBUG SenderThread:12305 [sender.py:send():382] send: config +2024-08-23 16:38:58,353 DEBUG SenderThread:12305 [sender.py:send():382] send: config +2024-08-23 16:38:58,807 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:39:00,353 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:00,808 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:39:04,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:39:04,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:39:04,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:39:06,113 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:11,113 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:16,114 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:19,926 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:39:19,926 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:39:19,967 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:39:21,162 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:21,819 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/config.yaml +2024-08-23 16:39:26,345 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:31,346 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:34,926 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:39:34,926 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:39:34,967 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:39:37,125 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:42,126 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:47,127 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:49,886 DEBUG SystemMonitor:12305 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-23 16:39:49,888 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:39:49,926 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:39:49,926 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:39:49,967 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:39:53,111 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:58,112 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:39:58,398 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:40:00,838 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:40:03,440 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:04,926 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:40:04,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:40:04,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:40:09,115 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:14,116 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:19,117 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:19,889 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:40:19,926 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:40:19,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:40:19,967 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:40:25,115 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:30,116 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:34,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:40:34,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:40:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:40:35,173 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:40,174 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:45,174 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:49,890 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:40:49,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:40:49,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:40:49,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:40:51,147 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:56,136 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:40:56,139 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:40:56,139 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:40:56,154 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:40:56,154 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:40:56,867 INFO Thread-12 :12305 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:40:58,868 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:41:01,180 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:04,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:41:04,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:41:04,929 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:41:07,173 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:12,174 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:17,174 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:19,891 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:41:19,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:41:19,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:41:19,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:41:23,138 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:28,138 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:33,139 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:34,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:41:34,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:41:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:41:39,118 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:44,119 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:49,120 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:49,892 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:41:49,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:41:49,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:41:49,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:41:53,851 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:41:53,852 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:41:53,852 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:41:53,854 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:41:53,896 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:41:54,892 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:41:54,897 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:41:59,892 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:04,893 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:04,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:42:04,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:42:04,929 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:42:10,105 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:15,106 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:19,893 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:42:19,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:42:19,927 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:42:19,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:42:20,146 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:25,146 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:30,147 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:34,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:42:34,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:42:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:42:36,138 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:41,138 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:46,139 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:49,894 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:42:49,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:42:49,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:42:49,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:42:51,557 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:42:51,559 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:42:51,559 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:42:51,560 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:42:51,561 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:42:51,926 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:42:52,927 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:42:56,600 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:01,601 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:04,927 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:43:04,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:43:04,930 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:43:07,132 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:12,132 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:17,132 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:19,895 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:43:19,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:43:19,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:43:19,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:43:23,107 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:28,108 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:33,108 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:34,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:43:34,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:43:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:43:38,202 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:43,203 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:48,203 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:49,180 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:43:49,181 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:43:49,182 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:43:49,184 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:43:49,896 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:43:49,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:43:49,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:43:49,930 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:43:49,954 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:43:50,955 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:43:54,116 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:43:59,116 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:04,117 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:04,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:44:04,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:44:04,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:44:09,123 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:14,124 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:19,124 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:19,897 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:44:19,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:44:19,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:44:19,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:44:25,120 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:30,120 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:34,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:44:34,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:44:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:44:35,144 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:40,145 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:45,145 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:46,917 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:44:46,920 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:44:46,921 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:44:46,922 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:44:46,983 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:44:48,984 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:44:49,898 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:44:49,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:44:49,928 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:44:49,931 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:44:50,159 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:44:55,160 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:00,160 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:04,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:45:04,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:45:04,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:45:06,142 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:11,142 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:16,142 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:19,899 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:45:19,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:45:19,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:45:19,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:45:21,199 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:26,199 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:31,200 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:34,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:45:34,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:45:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:45:37,098 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:42,098 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:44,582 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:45:44,584 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:45:44,584 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:45:44,585 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:45:45,011 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:45:47,011 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:45:47,624 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:49,899 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:45:49,928 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:45:49,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:45:49,931 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:45:53,153 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:45:58,153 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:03,154 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:04,929 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:46:04,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:46:04,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:46:09,099 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:14,100 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:19,101 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:19,900 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:46:19,929 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:46:19,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:46:19,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:46:24,145 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:29,146 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:34,146 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:34,929 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 16:46:34,929 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: stop_status +2024-08-23 16:46:34,971 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:46:40,145 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:42,271 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 16:46:42,273 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:46:42,273 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:46:42,274 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:46:43,038 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:46:45,038 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:46:45,316 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:46,081 DEBUG SenderThread:12305 [sender.py:send():382] send: exit +2024-08-23 16:46:46,081 INFO SenderThread:12305 [sender.py:send_exit():589] handling exit code: 255 +2024-08-23 16:46:46,081 INFO SenderThread:12305 [sender.py:send_exit():591] handling runtime: 476 +2024-08-23 16:46:46,083 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:46:46,083 INFO SenderThread:12305 [sender.py:send_exit():597] send defer +2024-08-23 16:46:46,083 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,083 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-23 16:46:46,083 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,083 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-23 16:46:46,083 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 1 +2024-08-23 16:46:46,083 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,083 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-23 16:46:46,084 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,084 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-23 16:46:46,084 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 2 +2024-08-23 16:46:46,084 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,084 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-23 16:46:46,084 INFO HandlerThread:12305 [system_monitor.py:finish():203] Stopping system monitor +2024-08-23 16:46:46,084 DEBUG SystemMonitor:12305 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-23 16:46:46,084 INFO HandlerThread:12305 [interfaces.py:finish():202] Joined cpu monitor +2024-08-23 16:46:46,084 DEBUG SystemMonitor:12305 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-23 16:46:46,084 INFO HandlerThread:12305 [interfaces.py:finish():202] Joined disk monitor +2024-08-23 16:46:46,118 INFO HandlerThread:12305 [interfaces.py:finish():202] Joined gpu monitor +2024-08-23 16:46:46,118 INFO HandlerThread:12305 [interfaces.py:finish():202] Joined memory monitor +2024-08-23 16:46:46,118 INFO HandlerThread:12305 [interfaces.py:finish():202] Joined network monitor +2024-08-23 16:46:46,118 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,119 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-23 16:46:46,119 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 3 +2024-08-23 16:46:46,119 DEBUG SenderThread:12305 [sender.py:send():382] send: stats +2024-08-23 16:46:46,119 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,119 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-23 16:46:46,120 DEBUG SenderThread:12305 [sender.py:send():382] send: history +2024-08-23 16:46:46,121 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: summary_record +2024-08-23 16:46:46,122 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:46:46,122 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,122 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-23 16:46:46,122 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 4 +2024-08-23 16:46:46,122 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,122 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-23 16:46:46,122 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,122 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-23 16:46:46,122 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 5 +2024-08-23 16:46:46,122 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,122 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-23 16:46:46,123 DEBUG SenderThread:12305 [sender.py:send():382] send: summary +2024-08-23 16:46:46,124 INFO SenderThread:12305 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 16:46:46,124 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,124 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-23 16:46:46,124 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 6 +2024-08-23 16:46:46,124 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,124 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-23 16:46:46,125 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,125 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-23 16:46:46,125 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 7 +2024-08-23 16:46:46,125 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 16:46:46,125 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:46,125 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-23 16:46:46,125 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:46,125 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-23 16:46:47,040 INFO Thread-12 :12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:46:47,081 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:47,096 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 8 +2024-08-23 16:46:47,096 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:47,096 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:47,097 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-23 16:46:47,097 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:47,097 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-23 16:46:47,097 INFO SenderThread:12305 [job_builder.py:build():296] Attempting to build job artifact +2024-08-23 16:46:47,098 INFO SenderThread:12305 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-23 16:46:47,113 INFO SenderThread:12305 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-23 16:46:47,121 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 9 +2024-08-23 16:46:47,122 DEBUG SenderThread:12305 [sender.py:send():382] send: artifact +2024-08-23 16:46:47,122 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:47,123 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-23 16:46:48,001 INFO SenderThread:12305 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'versionIndex': 3}}} +2024-08-23 16:46:48,001 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:48,001 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-23 16:46:48,001 INFO SenderThread:12305 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-23 16:46:48,041 INFO SenderThread:12305 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:46:48,041 INFO SenderThread:12305 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_163849-faey1t8u/files +2024-08-23 16:46:48,042 INFO SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/requirements.txt requirements.txt +2024-08-23 16:46:48,042 INFO SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/config.yaml config.yaml +2024-08-23 16:46:48,043 INFO SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-metadata.json wandb-metadata.json +2024-08-23 16:46:48,043 INFO SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json wandb-summary.json +2024-08-23 16:46:48,045 INFO SenderThread:12305 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_163849-faey1t8u/files/output.log output.log +2024-08-23 16:46:48,046 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 10 +2024-08-23 16:46:48,047 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:48,048 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-23 16:46:48,048 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:48,048 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-23 16:46:48,048 INFO SenderThread:12305 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:46:48,082 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:48,082 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:49,082 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:49,082 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:50,083 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:50,083 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:50,119 INFO wandb-upload_2:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/wandb-summary.json +2024-08-23 16:46:50,148 INFO wandb-upload_3:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/output.log +2024-08-23 16:46:50,166 INFO wandb-upload_0:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/requirements.txt +2024-08-23 16:46:50,204 INFO wandb-upload_1:12305 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_163849-faey1t8u/files/config.yaml +2024-08-23 16:46:50,405 INFO Thread-11 (_thread_body):12305 [sender.py:transition_state():617] send defer: 11 +2024-08-23 16:46:50,405 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:50,405 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-23 16:46:50,405 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:50,405 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-23 16:46:50,405 INFO SenderThread:12305 [file_pusher.py:join():178] waiting for file pusher +2024-08-23 16:46:50,405 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 12 +2024-08-23 16:46:50,405 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:50,406 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-23 16:46:50,406 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:50,406 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-23 16:46:50,406 INFO SenderThread:12305 [file_stream.py:finish():595] file stream finish called +2024-08-23 16:46:51,083 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:51,097 INFO SenderThread:12305 [file_stream.py:finish():599] file stream finish is done +2024-08-23 16:46:51,097 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 13 +2024-08-23 16:46:51,097 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:51,097 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:51,098 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-23 16:46:51,098 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:51,098 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-23 16:46:51,098 INFO SenderThread:12305 [sender.py:transition_state():617] send defer: 14 +2024-08-23 16:46:51,098 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: defer +2024-08-23 16:46:51,098 DEBUG SenderThread:12305 [sender.py:send():382] send: final +2024-08-23 16:46:51,098 INFO HandlerThread:12305 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-23 16:46:51,098 DEBUG SenderThread:12305 [sender.py:send():382] send: footer +2024-08-23 16:46:51,098 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: defer +2024-08-23 16:46:51,099 INFO SenderThread:12305 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-23 16:46:51,099 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:51,099 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:51,099 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 16:46:51,100 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: server_info +2024-08-23 16:46:51,100 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 16:46:51,100 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: get_summary +2024-08-23 16:46:51,100 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: server_info +2024-08-23 16:46:51,101 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-23 16:46:51,103 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 16:46:51,103 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: job_info +2024-08-23 16:46:51,261 DEBUG SenderThread:12305 [sender.py:send_request():409] send_request: job_info +2024-08-23 16:46:51,262 INFO MainThread:12305 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-23 16:46:51,262 INFO MainThread:12305 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-23 16:46:51,262 INFO MainThread:12305 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-23 16:46:51,263 DEBUG HandlerThread:12305 [handler.py:handle_request():146] handle_request: shutdown +2024-08-23 16:46:51,263 INFO HandlerThread:12305 [handler.py:finish():869] shutting down handler +2024-08-23 16:46:52,103 INFO WriterThread:12305 [datastore.py:close():296] close: /project/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb +2024-08-23 16:46:52,262 INFO SenderThread:12305 [sender.py:finish():1572] shutting down sender +2024-08-23 16:46:52,262 INFO SenderThread:12305 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 16:46:52,262 INFO SenderThread:12305 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240823_163849-faey1t8u/logs/debug.log b/wandb/run-20240823_163849-faey1t8u/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..129864654efbc3137417d88c94b655acbe5d24b4 --- /dev/null +++ b/wandb/run-20240823_163849-faey1t8u/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-23 16:38:49,358 INFO MainThread:12234 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-23 16:38:49,358 INFO MainThread:12234 [wandb_setup.py:_flush():76] Configure stats pid to 12234 +2024-08-23 16:38:49,358 INFO MainThread:12234 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-23 16:38:49,358 INFO MainThread:12234 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_163849-faey1t8u/logs/debug.log +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_163849-faey1t8u/logs/debug-internal.log +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_init.py:init():566] calling init triggers +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-16:38:35', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 7500, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 7500, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 3, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 106} +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_init.py:init():616] starting backend +2024-08-23 16:38:49,359 INFO MainThread:12234 [wandb_init.py:init():620] setting up manager +2024-08-23 16:38:49,364 INFO MainThread:12234 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-23 16:38:49,364 INFO MainThread:12234 [wandb_init.py:init():628] backend started and connected +2024-08-23 16:38:49,369 INFO MainThread:12234 [wandb_init.py:init():720] updated telemetry +2024-08-23 16:38:49,401 INFO MainThread:12234 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-23 16:38:49,805 INFO MainThread:12234 [wandb_run.py:_on_init():2262] communicating current version +2024-08-23 16:38:49,830 INFO MainThread:12234 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-23 16:38:49,830 INFO MainThread:12234 [wandb_init.py:init():804] starting run threads in backend +2024-08-23 16:38:49,926 INFO MainThread:12234 [wandb_run.py:_console_start():2241] atexit reg +2024-08-23 16:38:49,926 INFO MainThread:12234 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-23 16:38:49,926 INFO MainThread:12234 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-23 16:38:49,927 INFO MainThread:12234 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-23 16:38:49,927 INFO MainThread:12234 [wandb_init.py:init():847] run started, returning control to user process +2024-08-23 16:38:58,351 INFO MainThread:12234 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-23 16:38:58,352 INFO MainThread:12234 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-23 16:46:52,263 WARNING MsgRouterThr:12234 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb b/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb new file mode 100644 index 0000000000000000000000000000000000000000..65ff74fb2ce2e5bd87228e4ac1d40b91c5f7160d Binary files /dev/null and b/wandb/run-20240823_163849-faey1t8u/run-faey1t8u.wandb differ diff --git a/wandb/run-20240823_202540-om09pls8/files/config.yaml b/wandb/run-20240823_202540-om09pls8/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2c75e0365a472267e8957debd76870fb774340e --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +test_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: Qwen2-0.5b-0.2_train_2024-08-23-20:25:00 +wandb_project: + desc: null + value: llm_tutorial-0.2 +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +save: + desc: null + value: /work/llm_recipes/models/Qwen2-0.5b-0.2 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 3 +save_interval: + desc: null + value: 500 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 16000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 16000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 612 +micro_batch_size: + desc: null + value: 17 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/Qwen2-0.5b-0.2 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 10 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 4 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 9 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724412340.7504 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240823_202540-om09pls8/files/output.log b/wandb/run-20240823_202540-om09pls8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1f4c7ee69b333c3bb21eb86e9908f9f1e4e33517 --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/files/output.log @@ -0,0 +1,133 @@ +Created Hugging Face repository with ID koichi12/Qwen2-0.5b-0.2. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/model.pt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loaded model state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 9792000 + validation: 32644080 + test: 6120 +> building train, validation, and test datasets for GPT ... +Let split = None +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/optimizer.pt +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +[rank0]:[2024-08-23 20:26:02,460] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +Loaded optimizer state dict from /work/llm_recipes/models/Qwen2-0.5b-0.2/iter_0000100/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 23.617124223738323, Tokens per sec: 29262.917733714043, Loss: 3.7116615772247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 72.97505903558803, Tokens per sec: 90420.11757828314, Loss: 3.7358791828155518 +------------------------------------------------------------------ + eval ppl=30.41041374206543, eval loss=3.414785146713257 +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 65.5275483737983, Tokens per sec: 81192.2416628126, Loss: 3.757955551147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 73.41964709398604, Tokens per sec: 90970.9866703471, Loss: 3.730485439300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 70.30393549004248, Tokens per sec: 87110.44838107748, Loss: 3.7091140747070312 +------------------------------------------------------------------ + eval ppl=30.61298179626465, eval loss=3.421424150466919 +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 67.02992418712539, Tokens per sec: 83053.76804570397, Loss: 3.732792377471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 72.30676298983876, Tokens per sec: 89592.06196815638, Loss: 3.7457761764526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 73.24548732883933, Tokens per sec: 90755.19312868938, Loss: 3.7291133403778076 +------------------------------------------------------------------ + eval ppl=31.11376953125, eval loss=3.437650442123413 +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 66.08722155273428, Tokens per sec: 81885.7075580594, Loss: 3.71726131439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 73.33268637131981, Tokens per sec: 90863.23754520644, Loss: 3.7274892330169678 +------------------------------------------------------------------ +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 118, in train + loss.backward() + File "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py", line 522, in backward + torch.autograd.backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/__init__.py", line 267, in backward + _engine_run_backward( + File "/usr/local/lib/python3.10/dist-packages/torch/autograd/graph.py", line 681, in _engine_run_backward + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass +KeyboardInterrupt \ No newline at end of file diff --git a/wandb/run-20240823_202540-om09pls8/files/requirements.txt b/wandb/run-20240823_202540-om09pls8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json b/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d71e329bca51f6db5d46f8e468bb0fd241ff605f --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json @@ -0,0 +1,502 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-23T11:25:41.454442", + "startedAt": "2024-08-23T11:25:40.735970", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "17", + "--valid_micro_batch_size", + "10", + "--global-batch-size", + "612", + "--train-iters", + "16000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--test-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "16000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "500", + "--eval-interval", + "3", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--load", + "/work/llm_recipes/models/Qwen2-0.5b-0.2", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/Qwen2-0.5b-0.2", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial-0.2", + "--wandb-name", + "Qwen2-0.5b-0.2_train_2024-08-23-20:25:00" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 72, + "cpu_count_logical": 72, + "cpu_freq": { + "current": 2400.038999999999, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 226.66352462768555 + } +} diff --git a/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json b/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d1d440cc503172ab2f81ad91596310260aa0a71f --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 3.7274892330169678, "training/perplexity": 41.57459289701456, "utils/batch_size": 17, "utils/global_batch_size": 612, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 9, "utils/iteration": 110, "optimizer/lr": 5.18e-06, "optimizer/variance_l2": 0.04658828859035772, "optimizer/variance_sqrt_l2": 0.8728927373830674, "optimizer/momentum_l2": 0.8215464439288661, "optimizer/weight_l2": 640.8711356427281, "optimizer/variance_l1": 0.7605819702148438, "optimizer/variance_sqrt_l1": 1761.25, "optimizer/momentum_l1": 1269.75, "optimizer/weight_l1": 1809664.0, "optimizer/variance_abs_max": 0.04248046875, "optimizer/variance_sqrt_abs_max": 0.2060546875, "optimizer/momentum_abs_max": 0.18359375, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 6.903782178000256, "stats/tokens_per_sec": 90863.23754520644, "stats/tokens_per_sec_per_gpu": 22715.80938630161, "stats/tflops": 73.33268637131981, "_timestamp": 1724412471.451136, "_runtime": 130.7007360458374, "_step": 110, "evaluation/val_loss": 3.437650442123413, "evaluation/val_ppl": 31.11376953125, "_wandb": {"runtime": 131}} \ No newline at end of file diff --git a/wandb/run-20240823_202540-om09pls8/logs/debug-internal.log b/wandb/run-20240823_202540-om09pls8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8616de477ef846814449db211ffb1fe1d22aae1e --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/logs/debug-internal.log @@ -0,0 +1,312 @@ +2024-08-23 20:25:40,751 INFO StreamThr :13176 [internal.py:wandb_internal():86] W&B internal server running at pid: 13176, started at: 2024-08-23 20:25:40.750453 +2024-08-23 20:25:40,752 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status +2024-08-23 20:25:40,755 INFO WriterThread:13176 [datastore.py:open_for_write():87] open: /project/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb +2024-08-23 20:25:40,756 DEBUG SenderThread:13176 [sender.py:send():382] send: header +2024-08-23 20:25:40,869 DEBUG SenderThread:13176 [sender.py:send():382] send: run +2024-08-23 20:25:41,336 INFO SenderThread:13176 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240823_202540-om09pls8/files +2024-08-23 20:25:41,336 INFO SenderThread:13176 [sender.py:_start_run_threads():1136] run started: om09pls8 with start time 1724412340.7504 +2024-08-23 20:25:41,342 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: check_version +2024-08-23 20:25:41,342 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: check_version +2024-08-23 20:25:41,412 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: run_start +2024-08-23 20:25:41,418 DEBUG HandlerThread:13176 [system_info.py:__init__():27] System info init +2024-08-23 20:25:41,418 DEBUG HandlerThread:13176 [system_info.py:__init__():42] System info init done +2024-08-23 20:25:41,418 INFO HandlerThread:13176 [system_monitor.py:start():194] Starting system monitor +2024-08-23 20:25:41,418 INFO SystemMonitor:13176 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-23 20:25:41,418 INFO HandlerThread:13176 [system_monitor.py:probe():214] Collecting system info +2024-08-23 20:25:41,419 INFO SystemMonitor:13176 [interfaces.py:start():190] Started cpu monitoring +2024-08-23 20:25:41,419 INFO SystemMonitor:13176 [interfaces.py:start():190] Started disk monitoring +2024-08-23 20:25:41,420 INFO SystemMonitor:13176 [interfaces.py:start():190] Started gpu monitoring +2024-08-23 20:25:41,421 INFO SystemMonitor:13176 [interfaces.py:start():190] Started memory monitoring +2024-08-23 20:25:41,423 INFO SystemMonitor:13176 [interfaces.py:start():190] Started network monitoring +2024-08-23 20:25:41,454 DEBUG HandlerThread:13176 [system_info.py:probe():151] Probing system +2024-08-23 20:25:41,456 DEBUG HandlerThread:13176 [system_info.py:_probe_git():136] Probing git +2024-08-23 20:25:41,470 DEBUG HandlerThread:13176 [system_info.py:_probe_git():144] Probing git done +2024-08-23 20:25:41,470 DEBUG HandlerThread:13176 [system_info.py:probe():199] Probing system done +2024-08-23 20:25:41,470 DEBUG HandlerThread:13176 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-23T11:25:41.454442', 'startedAt': '2024-08-23T11:25:40.735970', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '17', '--valid_micro_batch_size', '10', '--global-batch-size', '612', '--train-iters', '16000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--test-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '16000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '500', '--eval-interval', '3', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--load', '/work/llm_recipes/models/Qwen2-0.5b-0.2', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/Qwen2-0.5b-0.2', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial-0.2', '--wandb-name', 'Qwen2-0.5b-0.2_train_2024-08-23-20:25:00'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 72, 'cpu_count_logical': 72, 'cpu_freq': {'current': 2400.038999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 226.66352462768555}} +2024-08-23 20:25:41,470 INFO HandlerThread:13176 [system_monitor.py:probe():224] Finished collecting system info +2024-08-23 20:25:41,470 INFO HandlerThread:13176 [system_monitor.py:probe():227] Publishing system info +2024-08-23 20:25:41,472 INFO HandlerThread:13176 [system_monitor.py:probe():229] Finished publishing system info +2024-08-23 20:25:41,502 DEBUG SenderThread:13176 [sender.py:send():382] send: files +2024-08-23 20:25:41,502 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-23 20:25:41,513 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: python_packages +2024-08-23 20:25:41,513 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:25:41,514 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:25:41,514 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: python_packages +2024-08-23 20:25:41,516 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:25:41,704 DEBUG SenderThread:13176 [sender.py:send():382] send: telemetry +2024-08-23 20:25:42,229 INFO wandb-upload_0:13176 [upload_job.py:push():131] Uploaded file /tmp/tmpnyk2zt9mwandb/u7uqpthk-wandb-metadata.json +2024-08-23 20:25:42,338 INFO Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:42,339 INFO Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json +2024-08-23 20:25:42,339 INFO Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/requirements.txt +2024-08-23 20:25:44,339 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:46,083 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:25:46,340 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:47,341 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:51,887 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:25:52,344 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:53,345 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:55,346 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:56,513 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:25:56,513 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:25:56,513 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:25:57,347 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:57,765 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:25:58,348 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:25:59,349 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:26:02,595 DEBUG SenderThread:13176 [sender.py:send():382] send: config +2024-08-23 20:26:02,595 DEBUG SenderThread:13176 [sender.py:send():382] send: config +2024-08-23 20:26:03,351 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:26:03,596 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:04,352 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:26:08,596 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:11,513 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:26:11,514 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:26:11,514 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:26:13,784 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:14,359 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/config.yaml +2024-08-23 20:26:18,968 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:23,969 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:28,970 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:30,283 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:26:30,422 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:26:30,422 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:26:34,601 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:39,602 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:39,739 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:26:41,424 DEBUG SystemMonitor:13176 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-23 20:26:41,427 DEBUG SenderThread:13176 [sender.py:send():382] send: stats +2024-08-23 20:26:42,374 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:26:45,083 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:26:45,084 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:26:45,084 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:45,085 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:26:46,679 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:26:46,682 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:26:46,682 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:26:46,684 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:26:47,378 INFO Thread-12 :13176 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:26:48,379 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:26:49,094 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:26:50,093 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:50,380 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:26:55,094 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:26:56,822 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:26:56,825 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:26:56,826 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:26:56,829 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:26:57,385 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:26:58,386 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:00,084 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:27:00,085 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:27:00,085 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:27:00,326 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:03,720 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:03,722 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:03,722 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:03,723 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:04,390 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:05,764 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:06,392 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:10,765 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:10,923 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:10,926 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:10,926 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:10,928 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:11,395 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:11,429 DEBUG SenderThread:13176 [sender.py:send():382] send: stats +2024-08-23 20:27:12,396 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:13,173 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:14,397 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:15,084 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:27:15,084 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:27:15,085 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:27:16,306 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:20,728 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:20,730 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:20,730 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:20,731 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:21,402 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:21,772 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:22,403 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:26,773 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:27,731 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:27,734 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:27,734 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:27,736 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:28,408 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:30,084 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:27:30,085 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:27:30,085 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:27:30,409 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:32,346 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:34,646 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:34,649 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:34,649 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:34,651 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:35,413 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:36,413 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:36,883 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:37,883 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:38,415 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:41,432 DEBUG SenderThread:13176 [sender.py:send():382] send: stats +2024-08-23 20:27:43,433 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:44,546 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:44,549 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:44,549 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:44,551 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:45,085 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: stop_status +2024-08-23 20:27:45,085 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:27:45,085 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: stop_status +2024-08-23 20:27:45,420 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:46,421 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:49,351 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:51,452 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: partial_history +2024-08-23 20:27:51,454 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:51,454 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:51,456 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:52,425 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:53,012 DEBUG SenderThread:13176 [sender.py:send():382] send: exit +2024-08-23 20:27:53,012 INFO SenderThread:13176 [sender.py:send_exit():589] handling exit code: 255 +2024-08-23 20:27:53,013 INFO SenderThread:13176 [sender.py:send_exit():591] handling runtime: 131 +2024-08-23 20:27:53,014 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:53,014 INFO SenderThread:13176 [sender.py:send_exit():597] send defer +2024-08-23 20:27:53,014 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,014 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-23 20:27:53,015 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,015 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-23 20:27:53,015 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 1 +2024-08-23 20:27:53,015 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,015 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-23 20:27:53,015 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,015 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-23 20:27:53,015 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 2 +2024-08-23 20:27:53,015 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,015 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-23 20:27:53,015 INFO HandlerThread:13176 [system_monitor.py:finish():203] Stopping system monitor +2024-08-23 20:27:53,015 DEBUG SystemMonitor:13176 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-23 20:27:53,016 DEBUG SystemMonitor:13176 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-23 20:27:53,016 INFO HandlerThread:13176 [interfaces.py:finish():202] Joined cpu monitor +2024-08-23 20:27:53,018 INFO HandlerThread:13176 [interfaces.py:finish():202] Joined disk monitor +2024-08-23 20:27:53,348 INFO HandlerThread:13176 [interfaces.py:finish():202] Joined gpu monitor +2024-08-23 20:27:53,349 INFO HandlerThread:13176 [interfaces.py:finish():202] Joined memory monitor +2024-08-23 20:27:53,349 INFO HandlerThread:13176 [interfaces.py:finish():202] Joined network monitor +2024-08-23 20:27:53,350 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,350 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-23 20:27:53,350 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 3 +2024-08-23 20:27:53,350 DEBUG SenderThread:13176 [sender.py:send():382] send: stats +2024-08-23 20:27:53,350 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,351 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-23 20:27:53,354 DEBUG SenderThread:13176 [sender.py:send():382] send: history +2024-08-23 20:27:53,354 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: summary_record +2024-08-23 20:27:53,355 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:53,355 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,355 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-23 20:27:53,356 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 4 +2024-08-23 20:27:53,356 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,356 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-23 20:27:53,356 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,356 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-23 20:27:53,356 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 5 +2024-08-23 20:27:53,356 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,356 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-23 20:27:53,358 DEBUG SenderThread:13176 [sender.py:send():382] send: summary +2024-08-23 20:27:53,358 INFO SenderThread:13176 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-23 20:27:53,359 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,359 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-23 20:27:53,359 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 6 +2024-08-23 20:27:53,359 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,359 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-23 20:27:53,359 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,359 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-23 20:27:53,359 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 7 +2024-08-23 20:27:53,359 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: status_report +2024-08-23 20:27:53,360 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:53,360 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-23 20:27:53,360 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:53,360 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-23 20:27:53,427 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:54,012 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 20:27:54,427 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:54,598 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 8 +2024-08-23 20:27:54,598 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 20:27:54,598 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:54,598 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-23 20:27:54,599 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:54,599 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-23 20:27:54,599 INFO SenderThread:13176 [job_builder.py:build():296] Attempting to build job artifact +2024-08-23 20:27:54,600 INFO SenderThread:13176 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-23 20:27:54,620 INFO SenderThread:13176 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-23 20:27:54,630 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 9 +2024-08-23 20:27:54,631 DEBUG SenderThread:13176 [sender.py:send():382] send: artifact +2024-08-23 20:27:54,631 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:54,632 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-23 20:27:55,013 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 20:27:55,428 INFO Thread-12 :13176 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:55,507 INFO SenderThread:13176 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MTk4ODkxMQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjQ1ODQ1MA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MjAxODA1Mw==', 'versionIndex': 3}}} +2024-08-23 20:27:55,507 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:55,507 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-23 20:27:55,507 INFO SenderThread:13176 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-23 20:27:56,429 INFO SenderThread:13176 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240823_202540-om09pls8/files +2024-08-23 20:27:56,429 INFO SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/requirements.txt requirements.txt +2024-08-23 20:27:56,429 INFO SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/config.yaml config.yaml +2024-08-23 20:27:56,431 INFO SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/wandb-metadata.json wandb-metadata.json +2024-08-23 20:27:56,431 INFO SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json wandb-summary.json +2024-08-23 20:27:56,433 INFO SenderThread:13176 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240823_202540-om09pls8/files/output.log output.log +2024-08-23 20:27:56,433 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 10 +2024-08-23 20:27:56,434 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 20:27:56,435 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:56,435 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-23 20:27:56,436 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:56,436 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-23 20:27:56,436 INFO SenderThread:13176 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 20:27:56,841 INFO wandb-upload_0:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/requirements.txt +2024-08-23 20:27:56,894 INFO wandb-upload_2:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/wandb-summary.json +2024-08-23 20:27:56,896 INFO wandb-upload_3:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/output.log +2024-08-23 20:27:56,902 INFO wandb-upload_1:13176 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240823_202540-om09pls8/files/config.yaml +2024-08-23 20:27:57,013 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 20:27:57,014 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 20:27:57,103 INFO Thread-11 (_thread_body):13176 [sender.py:transition_state():617] send defer: 11 +2024-08-23 20:27:57,103 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:57,103 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-23 20:27:57,103 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:57,103 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-23 20:27:57,103 INFO SenderThread:13176 [file_pusher.py:join():178] waiting for file pusher +2024-08-23 20:27:57,103 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 12 +2024-08-23 20:27:57,103 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:57,103 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-23 20:27:57,104 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:57,104 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-23 20:27:57,104 INFO SenderThread:13176 [file_stream.py:finish():595] file stream finish called +2024-08-23 20:27:57,389 INFO SenderThread:13176 [file_stream.py:finish():599] file stream finish is done +2024-08-23 20:27:57,389 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 13 +2024-08-23 20:27:57,389 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:57,389 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-23 20:27:57,389 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:57,389 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-23 20:27:57,389 INFO SenderThread:13176 [sender.py:transition_state():617] send defer: 14 +2024-08-23 20:27:57,390 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: defer +2024-08-23 20:27:57,390 DEBUG SenderThread:13176 [sender.py:send():382] send: final +2024-08-23 20:27:57,390 INFO HandlerThread:13176 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-23 20:27:57,390 DEBUG SenderThread:13176 [sender.py:send():382] send: footer +2024-08-23 20:27:57,390 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: defer +2024-08-23 20:27:57,390 INFO SenderThread:13176 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-23 20:27:57,391 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 20:27:57,391 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 20:27:57,391 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-23 20:27:57,391 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: server_info +2024-08-23 20:27:57,392 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: poll_exit +2024-08-23 20:27:57,392 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: get_summary +2024-08-23 20:27:57,392 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: server_info +2024-08-23 20:27:57,393 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-23 20:27:57,395 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-23 20:27:57,395 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: job_info +2024-08-23 20:27:57,563 DEBUG SenderThread:13176 [sender.py:send_request():409] send_request: job_info +2024-08-23 20:27:57,563 INFO MainThread:13176 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-23 20:27:57,564 INFO MainThread:13176 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-23 20:27:57,564 INFO MainThread:13176 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-23 20:27:57,564 DEBUG HandlerThread:13176 [handler.py:handle_request():146] handle_request: shutdown +2024-08-23 20:27:57,564 INFO HandlerThread:13176 [handler.py:finish():869] shutting down handler +2024-08-23 20:27:58,396 INFO WriterThread:13176 [datastore.py:close():296] close: /project/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb +2024-08-23 20:27:58,563 INFO SenderThread:13176 [sender.py:finish():1572] shutting down sender +2024-08-23 20:27:58,563 INFO SenderThread:13176 [file_pusher.py:finish():172] shutting down file pusher +2024-08-23 20:27:58,563 INFO SenderThread:13176 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240823_202540-om09pls8/logs/debug.log b/wandb/run-20240823_202540-om09pls8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5b5e17a8063b4e024622ce3dba1e662ab058b01c --- /dev/null +++ b/wandb/run-20240823_202540-om09pls8/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-23 20:25:40,742 INFO MainThread:12857 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_setup.py:_flush():76] Configure stats pid to 12857 +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240823_202540-om09pls8/logs/debug.log +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240823_202540-om09pls8/logs/debug-internal.log +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_init.py:init():566] calling init triggers +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'test_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'Qwen2-0.5b-0.2_train_2024-08-23-20:25:00', 'wandb_project': 'llm_tutorial-0.2', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'save': '/work/llm_recipes/models/Qwen2-0.5b-0.2', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 3, 'save_interval': 500, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 16000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 16000, 'train_samples': None, 'global_batch_size': 612, 'micro_batch_size': 17, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/Qwen2-0.5b-0.2', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 10, 'rank': 0, 'world_size': 4, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 9} +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_init.py:init():616] starting backend +2024-08-23 20:25:40,743 INFO MainThread:12857 [wandb_init.py:init():620] setting up manager +2024-08-23 20:25:40,748 INFO MainThread:12857 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-23 20:25:40,750 INFO MainThread:12857 [wandb_init.py:init():628] backend started and connected +2024-08-23 20:25:40,754 INFO MainThread:12857 [wandb_init.py:init():720] updated telemetry +2024-08-23 20:25:40,864 INFO MainThread:12857 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-23 20:25:41,341 INFO MainThread:12857 [wandb_run.py:_on_init():2262] communicating current version +2024-08-23 20:25:41,364 INFO MainThread:12857 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-23 20:25:41,364 INFO MainThread:12857 [wandb_init.py:init():804] starting run threads in backend +2024-08-23 20:25:41,512 INFO MainThread:12857 [wandb_run.py:_console_start():2241] atexit reg +2024-08-23 20:25:41,512 INFO MainThread:12857 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-23 20:25:41,512 INFO MainThread:12857 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-23 20:25:41,513 INFO MainThread:12857 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-23 20:25:41,514 INFO MainThread:12857 [wandb_init.py:init():847] run started, returning control to user process +2024-08-23 20:26:02,594 INFO MainThread:12857 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-23 20:26:02,595 INFO MainThread:12857 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 4} +2024-08-23 20:27:58,565 WARNING MsgRouterThr:12857 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb b/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f03d1684d5351779ad5a13b80136bf45117deb0e Binary files /dev/null and b/wandb/run-20240823_202540-om09pls8/run-om09pls8.wandb differ diff --git a/wandb/run-20240831_192346-5vo4p2k7/files/config.yaml b/wandb/run-20240831_192346-5vo4p2k7/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47032bc6b93657be9fdad8fc12bcbe19b6844d1c --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/files/config.yaml @@ -0,0 +1,313 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: SHARD_GRAD_OP +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: null +valid_data_path: + desc: null + value: null +test_data_path: + desc: null + value: null +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 2048 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:23:33 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 100 +save_interval: + desc: null + value: 100 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 1000 +lr_warmup_iters: + desc: null + value: 25 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 1000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 16 +micro_batch_size: + desc: null + value: 4 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.99 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +instruction_train_data_path: + desc: null + value: /work/datasets/bin/ichikara/train/data +instruction_valid_data_path: + desc: null + value: /work/datasets/bin/ichikara/valid/data +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: true +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: source +target_key: + desc: null + value: target +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: true +remove_padding_masking: + desc: null + value: true +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 4 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1725099826.21653 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20240831_192346-5vo4p2k7/files/output.log b/wandb/run-20240831_192346-5vo4p2k7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..390217e6ada39c61da01d5896d596c95cb92fb2c --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/files/output.log @@ -0,0 +1,15 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 103, in main + model = get_model( + File "/project/src/llama_recipes/get_models.py", line 106, in get_model + assert sliding_window == 131072 +AssertionError \ No newline at end of file diff --git a/wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt b/wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240831_192346-5vo4p2k7/files/wandb-metadata.json b/wandb/run-20240831_192346-5vo4p2k7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0e5ae504c478415b6d6bd5282d931d399fe67e2f --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/files/wandb-metadata.json @@ -0,0 +1,221 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-31T10:23:46.789150", + "startedAt": "2024-08-31T10:23:46.204215", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "2048", + "--micro-batch-size", + "4", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "16", + "--train-iters", + "1000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--instruction-train-data-path", + "/work/datasets/bin/ichikara/train/data", + "--instruction-valid-data-path", + "/work/datasets/bin/ichikara/valid/data", + "--efficient-instruction-tuning", + "--remove-padding-masking", + "--source-key", + "source", + "--target-key", + "target", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "25", + "--lr-decay-iters", + "1000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.99", + "--adam-eps", + "1e-6", + "--save-interval", + "100", + "--eval-interval", + "100", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--hf-transformer-model-dir", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "SHARD_GRAD_OP", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-sampler-state", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara", + "--num-workers", + "4", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:23:33" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.025999999999, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48781967163086 + } +} diff --git a/wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json b/wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac1ba993956be52f459e677e193218ff2cddc76 --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 3}} \ No newline at end of file diff --git a/wandb/run-20240831_192346-5vo4p2k7/logs/debug-internal.log b/wandb/run-20240831_192346-5vo4p2k7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..40303290dcdbd8418007a0cb8678fb3ae6cd6f4d --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/logs/debug-internal.log @@ -0,0 +1,188 @@ +2024-08-31 19:23:46,218 INFO StreamThr :13619 [internal.py:wandb_internal():86] W&B internal server running at pid: 13619, started at: 2024-08-31 19:23:46.217947 +2024-08-31 19:23:46,220 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: status +2024-08-31 19:23:46,221 INFO WriterThread:13619 [datastore.py:open_for_write():87] open: /project/wandb/run-20240831_192346-5vo4p2k7/run-5vo4p2k7.wandb +2024-08-31 19:23:46,222 DEBUG SenderThread:13619 [sender.py:send():382] send: header +2024-08-31 19:23:46,236 DEBUG SenderThread:13619 [sender.py:send():382] send: run +2024-08-31 19:23:46,695 INFO SenderThread:13619 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240831_192346-5vo4p2k7/files +2024-08-31 19:23:46,695 INFO SenderThread:13619 [sender.py:_start_run_threads():1136] run started: 5vo4p2k7 with start time 1725099826.21653 +2024-08-31 19:23:46,702 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: check_version +2024-08-31 19:23:46,702 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: check_version +2024-08-31 19:23:46,770 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: run_start +2024-08-31 19:23:46,776 DEBUG HandlerThread:13619 [system_info.py:__init__():27] System info init +2024-08-31 19:23:46,776 DEBUG HandlerThread:13619 [system_info.py:__init__():42] System info init done +2024-08-31 19:23:46,776 INFO HandlerThread:13619 [system_monitor.py:start():194] Starting system monitor +2024-08-31 19:23:46,776 INFO SystemMonitor:13619 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-31 19:23:46,776 INFO HandlerThread:13619 [system_monitor.py:probe():214] Collecting system info +2024-08-31 19:23:46,777 INFO SystemMonitor:13619 [interfaces.py:start():190] Started cpu monitoring +2024-08-31 19:23:46,777 INFO SystemMonitor:13619 [interfaces.py:start():190] Started disk monitoring +2024-08-31 19:23:46,778 INFO SystemMonitor:13619 [interfaces.py:start():190] Started gpu monitoring +2024-08-31 19:23:46,779 INFO SystemMonitor:13619 [interfaces.py:start():190] Started memory monitoring +2024-08-31 19:23:46,780 INFO SystemMonitor:13619 [interfaces.py:start():190] Started network monitoring +2024-08-31 19:23:46,789 DEBUG HandlerThread:13619 [system_info.py:probe():151] Probing system +2024-08-31 19:23:46,792 DEBUG HandlerThread:13619 [system_info.py:_probe_git():136] Probing git +2024-08-31 19:23:46,804 DEBUG HandlerThread:13619 [system_info.py:_probe_git():144] Probing git done +2024-08-31 19:23:46,804 DEBUG HandlerThread:13619 [system_info.py:probe():199] Probing system done +2024-08-31 19:23:46,804 DEBUG HandlerThread:13619 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-31T10:23:46.789150', 'startedAt': '2024-08-31T10:23:46.204215', 'docker': None, 'cuda': None, 'args': ('--seq-length', '2048', '--micro-batch-size', '4', '--valid_micro_batch_size', '1', '--global-batch-size', '16', '--train-iters', '1000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--instruction-train-data-path', '/work/datasets/bin/ichikara/train/data', '--instruction-valid-data-path', '/work/datasets/bin/ichikara/valid/data', '--efficient-instruction-tuning', '--remove-padding-masking', '--source-key', 'source', '--target-key', 'target', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '25', '--lr-decay-iters', '1000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.99', '--adam-eps', '1e-6', '--save-interval', '100', '--eval-interval', '100', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--hf-transformer-model-dir', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', '--fsdp-activation-checkpointing', '--sharding-strategy', 'SHARD_GRAD_OP', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-sampler-state', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', '--num-workers', '4', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:23:33'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.025999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48781967163086}} +2024-08-31 19:23:46,804 INFO HandlerThread:13619 [system_monitor.py:probe():224] Finished collecting system info +2024-08-31 19:23:46,804 INFO HandlerThread:13619 [system_monitor.py:probe():227] Publishing system info +2024-08-31 19:23:46,806 INFO HandlerThread:13619 [system_monitor.py:probe():229] Finished publishing system info +2024-08-31 19:23:46,833 DEBUG SenderThread:13619 [sender.py:send():382] send: files +2024-08-31 19:23:46,833 INFO SenderThread:13619 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-31 19:23:46,845 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: python_packages +2024-08-31 19:23:46,845 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 19:23:46,845 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 19:23:46,846 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: python_packages +2024-08-31 19:23:46,848 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: stop_status +2024-08-31 19:23:47,025 DEBUG SenderThread:13619 [sender.py:send():382] send: telemetry +2024-08-31 19:23:47,542 INFO wandb-upload_0:13619 [upload_job.py:push():131] Uploaded file /tmp/tmp1bcrzhapwandb/3v185don-wandb-metadata.json +2024-08-31 19:23:47,697 INFO Thread-12 :13619 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt +2024-08-31 19:23:47,697 INFO Thread-12 :13619 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192346-5vo4p2k7/files/wandb-metadata.json +2024-08-31 19:23:49,698 INFO Thread-12 :13619 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192346-5vo4p2k7/files/output.log +2024-08-31 19:23:50,201 DEBUG SenderThread:13619 [sender.py:send():382] send: exit +2024-08-31 19:23:50,202 INFO SenderThread:13619 [sender.py:send_exit():589] handling exit code: 1 +2024-08-31 19:23:50,202 INFO SenderThread:13619 [sender.py:send_exit():591] handling runtime: 3 +2024-08-31 19:23:50,204 INFO SenderThread:13619 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 19:23:50,204 INFO SenderThread:13619 [sender.py:send_exit():597] send defer +2024-08-31 19:23:50,204 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,204 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-31 19:23:50,204 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,204 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-31 19:23:50,204 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 1 +2024-08-31 19:23:50,204 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,204 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-31 19:23:50,205 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,205 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-31 19:23:50,205 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 2 +2024-08-31 19:23:50,205 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,205 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-31 19:23:50,205 INFO HandlerThread:13619 [system_monitor.py:finish():203] Stopping system monitor +2024-08-31 19:23:50,205 INFO HandlerThread:13619 [interfaces.py:finish():202] Joined cpu monitor +2024-08-31 19:23:50,205 DEBUG SystemMonitor:13619 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-31 19:23:50,205 INFO HandlerThread:13619 [interfaces.py:finish():202] Joined disk monitor +2024-08-31 19:23:50,206 DEBUG SystemMonitor:13619 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-31 19:23:50,206 DEBUG SystemMonitor:13619 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-31 19:23:50,239 INFO HandlerThread:13619 [interfaces.py:finish():202] Joined gpu monitor +2024-08-31 19:23:50,239 INFO HandlerThread:13619 [interfaces.py:finish():202] Joined memory monitor +2024-08-31 19:23:50,239 INFO HandlerThread:13619 [interfaces.py:finish():202] Joined network monitor +2024-08-31 19:23:50,240 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,240 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-31 19:23:50,240 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 3 +2024-08-31 19:23:50,240 DEBUG SenderThread:13619 [sender.py:send():382] send: stats +2024-08-31 19:23:50,240 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,240 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-31 19:23:50,240 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,240 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-31 19:23:50,240 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 4 +2024-08-31 19:23:50,240 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,240 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-31 19:23:50,241 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,241 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-31 19:23:50,241 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 5 +2024-08-31 19:23:50,241 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,241 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-31 19:23:50,241 DEBUG SenderThread:13619 [sender.py:send():382] send: summary +2024-08-31 19:23:50,242 INFO SenderThread:13619 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 19:23:50,242 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,242 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-31 19:23:50,242 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 6 +2024-08-31 19:23:50,242 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,242 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-31 19:23:50,242 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,243 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-31 19:23:50,245 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 19:23:50,440 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 7 +2024-08-31 19:23:50,440 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:50,441 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-31 19:23:50,441 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:50,441 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-31 19:23:50,699 INFO Thread-12 :13619 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_192346-5vo4p2k7/files/config.yaml +2024-08-31 19:23:50,700 INFO Thread-12 :13619 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json +2024-08-31 19:23:51,202 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:23:51,700 INFO Thread-12 :13619 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_192346-5vo4p2k7/files/output.log +2024-08-31 19:23:52,192 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 8 +2024-08-31 19:23:52,192 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:23:52,192 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:52,192 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-31 19:23:52,192 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:52,193 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-31 19:23:52,193 INFO SenderThread:13619 [job_builder.py:build():296] Attempting to build job artifact +2024-08-31 19:23:52,193 INFO SenderThread:13619 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-31 19:23:52,202 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:23:52,210 INFO SenderThread:13619 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-31 19:23:52,219 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 9 +2024-08-31 19:23:52,219 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:23:52,219 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:52,220 DEBUG SenderThread:13619 [sender.py:send():382] send: artifact +2024-08-31 19:23:52,220 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-31 19:23:52,701 INFO Thread-12 :13619 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_192346-5vo4p2k7/files/output.log +2024-08-31 19:23:53,203 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:23:53,368 INFO wandb-upload_1:13619 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmptu7tlh0q +2024-08-31 19:23:54,403 INFO wandb-upload_0:13619 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmp9ja5qyty +2024-08-31 19:23:55,551 INFO SenderThread:13619 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE3NDAxOTIzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjgwMzg3NA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2NDgyMTAwMw==', 'versionIndex': 1}}} +2024-08-31 19:23:55,552 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:55,552 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-31 19:23:55,552 INFO SenderThread:13619 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-31 19:23:55,702 INFO SenderThread:13619 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240831_192346-5vo4p2k7/files +2024-08-31 19:23:55,702 INFO SenderThread:13619 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt requirements.txt +2024-08-31 19:23:55,702 INFO SenderThread:13619 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192346-5vo4p2k7/files/config.yaml config.yaml +2024-08-31 19:23:55,704 INFO SenderThread:13619 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192346-5vo4p2k7/files/wandb-metadata.json wandb-metadata.json +2024-08-31 19:23:55,704 INFO SenderThread:13619 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json wandb-summary.json +2024-08-31 19:23:55,706 INFO SenderThread:13619 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192346-5vo4p2k7/files/output.log output.log +2024-08-31 19:23:55,707 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 10 +2024-08-31 19:23:55,707 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:23:55,708 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:55,708 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-31 19:23:55,710 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:55,710 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-31 19:23:55,710 INFO SenderThread:13619 [file_pusher.py:finish():172] shutting down file pusher +2024-08-31 19:23:56,130 INFO wandb-upload_2:13619 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192346-5vo4p2k7/files/wandb-summary.json +2024-08-31 19:23:56,130 INFO wandb-upload_0:13619 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192346-5vo4p2k7/files/config.yaml +2024-08-31 19:23:56,204 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:23:56,204 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:23:56,238 INFO wandb-upload_3:13619 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192346-5vo4p2k7/files/output.log +2024-08-31 19:23:56,244 INFO wandb-upload_1:13619 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192346-5vo4p2k7/files/requirements.txt +2024-08-31 19:23:56,444 INFO Thread-11 (_thread_body):13619 [sender.py:transition_state():617] send defer: 11 +2024-08-31 19:23:56,445 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:56,445 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-31 19:23:56,445 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:56,445 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-31 19:23:56,445 INFO SenderThread:13619 [file_pusher.py:join():178] waiting for file pusher +2024-08-31 19:23:56,445 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 12 +2024-08-31 19:23:56,445 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:56,445 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-31 19:23:56,446 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:56,446 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-31 19:23:56,446 INFO SenderThread:13619 [file_stream.py:finish():595] file stream finish called +2024-08-31 19:23:56,617 INFO SenderThread:13619 [file_stream.py:finish():599] file stream finish is done +2024-08-31 19:23:56,618 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 13 +2024-08-31 19:23:56,618 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:56,618 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-31 19:23:56,618 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:56,618 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-31 19:23:56,618 INFO SenderThread:13619 [sender.py:transition_state():617] send defer: 14 +2024-08-31 19:23:56,619 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:23:56,619 DEBUG SenderThread:13619 [sender.py:send():382] send: final +2024-08-31 19:23:56,619 INFO HandlerThread:13619 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-31 19:23:56,619 DEBUG SenderThread:13619 [sender.py:send():382] send: footer +2024-08-31 19:23:56,619 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: defer +2024-08-31 19:23:56,619 INFO SenderThread:13619 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-31 19:23:56,619 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:23:56,620 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:23:56,620 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:23:56,620 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: server_info +2024-08-31 19:23:56,621 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:23:56,621 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: get_summary +2024-08-31 19:23:56,621 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: server_info +2024-08-31 19:23:56,621 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-31 19:23:56,623 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 19:23:56,623 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: job_info +2024-08-31 19:23:56,788 DEBUG SenderThread:13619 [sender.py:send_request():409] send_request: job_info +2024-08-31 19:23:56,788 INFO MainThread:13619 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-31 19:23:56,788 INFO MainThread:13619 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-31 19:23:56,788 INFO MainThread:13619 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-31 19:23:56,789 DEBUG HandlerThread:13619 [handler.py:handle_request():146] handle_request: shutdown +2024-08-31 19:23:56,789 INFO HandlerThread:13619 [handler.py:finish():869] shutting down handler +2024-08-31 19:23:57,623 INFO WriterThread:13619 [datastore.py:close():296] close: /project/wandb/run-20240831_192346-5vo4p2k7/run-5vo4p2k7.wandb +2024-08-31 19:23:57,788 INFO SenderThread:13619 [sender.py:finish():1572] shutting down sender +2024-08-31 19:23:57,788 INFO SenderThread:13619 [file_pusher.py:finish():172] shutting down file pusher +2024-08-31 19:23:57,788 INFO SenderThread:13619 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240831_192346-5vo4p2k7/logs/debug.log b/wandb/run-20240831_192346-5vo4p2k7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6fcd6f72dc36121078539d5b89eeed34b4a9f2e2 --- /dev/null +++ b/wandb/run-20240831_192346-5vo4p2k7/logs/debug.log @@ -0,0 +1,28 @@ +2024-08-31 19:23:46,209 INFO MainThread:13548 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_setup.py:_flush():76] Configure stats pid to 13548 +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240831_192346-5vo4p2k7/logs/debug.log +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240831_192346-5vo4p2k7/logs/debug-internal.log +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_init.py:init():566] calling init triggers +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'SHARD_GRAD_OP', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': None, 'valid_data_path': None, 'test_data_path': None, 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 2048, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:23:33', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 100, 'save_interval': 100, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 1000, 'lr_warmup_iters': 25, 'min_lr': 1e-06, 'train_iters': 1000, 'train_samples': None, 'global_batch_size': 16, 'micro_batch_size': 4, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.99, 'adam_eps': 1e-06, 'hf_transformer_model_dir': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'instruction_train_data_path': '/work/datasets/bin/ichikara/train/data', 'instruction_valid_data_path': '/work/datasets/bin/ichikara/valid/data', 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': True, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': 'source', 'target_key': 'target', 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': True, 'remove_padding_masking': True, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 4} +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_init.py:init():616] starting backend +2024-08-31 19:23:46,210 INFO MainThread:13548 [wandb_init.py:init():620] setting up manager +2024-08-31 19:23:46,215 INFO MainThread:13548 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-31 19:23:46,216 INFO MainThread:13548 [wandb_init.py:init():628] backend started and connected +2024-08-31 19:23:46,221 INFO MainThread:13548 [wandb_init.py:init():720] updated telemetry +2024-08-31 19:23:46,231 INFO MainThread:13548 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-31 19:23:46,701 INFO MainThread:13548 [wandb_run.py:_on_init():2262] communicating current version +2024-08-31 19:23:46,724 INFO MainThread:13548 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.8 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-31 19:23:46,724 INFO MainThread:13548 [wandb_init.py:init():804] starting run threads in backend +2024-08-31 19:23:46,844 INFO MainThread:13548 [wandb_run.py:_console_start():2241] atexit reg +2024-08-31 19:23:46,844 INFO MainThread:13548 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-31 19:23:46,844 INFO MainThread:13548 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-31 19:23:46,844 INFO MainThread:13548 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-31 19:23:46,845 INFO MainThread:13548 [wandb_init.py:init():847] run started, returning control to user process +2024-08-31 19:23:57,789 WARNING MsgRouterThr:13548 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240831_192346-5vo4p2k7/run-5vo4p2k7.wandb b/wandb/run-20240831_192346-5vo4p2k7/run-5vo4p2k7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8b1421981e83c68c2d13db4b1ae023b77471321f Binary files /dev/null and b/wandb/run-20240831_192346-5vo4p2k7/run-5vo4p2k7.wandb differ diff --git a/wandb/run-20240831_192718-8hc38ffs/files/config.yaml b/wandb/run-20240831_192718-8hc38ffs/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9966e8bc3b9a32552f7f2bb647319c97a9d7c907 --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/files/config.yaml @@ -0,0 +1,313 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: SHARD_GRAD_OP +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: null +valid_data_path: + desc: null + value: null +test_data_path: + desc: null + value: null +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:27:01 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 100 +save_interval: + desc: null + value: 100 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 1000 +lr_warmup_iters: + desc: null + value: 25 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 1000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 16 +micro_batch_size: + desc: null + value: 4 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.99 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +instruction_train_data_path: + desc: null + value: /work/datasets/bin/ichikara/train/data +instruction_valid_data_path: + desc: null + value: /work/datasets/bin/ichikara/valid/data +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: true +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: source +target_key: + desc: null + value: target +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: true +remove_padding_masking: + desc: null + value: true +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 4 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1725100038.866512 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20240831_192718-8hc38ffs/files/output.log b/wandb/run-20240831_192718-8hc38ffs/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..390217e6ada39c61da01d5896d596c95cb92fb2c --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/files/output.log @@ -0,0 +1,15 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara/latest_iteration.txt +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 103, in main + model = get_model( + File "/project/src/llama_recipes/get_models.py", line 106, in get_model + assert sliding_window == 131072 +AssertionError \ No newline at end of file diff --git a/wandb/run-20240831_192718-8hc38ffs/files/requirements.txt b/wandb/run-20240831_192718-8hc38ffs/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240831_192718-8hc38ffs/files/wandb-metadata.json b/wandb/run-20240831_192718-8hc38ffs/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..17a5bc84d769f74e7c2a5906f391679506c42163 --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/files/wandb-metadata.json @@ -0,0 +1,221 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-31T10:27:19.369639", + "startedAt": "2024-08-31T10:27:18.853756", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--micro-batch-size", + "4", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "16", + "--train-iters", + "1000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--instruction-train-data-path", + "/work/datasets/bin/ichikara/train/data", + "--instruction-valid-data-path", + "/work/datasets/bin/ichikara/valid/data", + "--efficient-instruction-tuning", + "--remove-padding-masking", + "--source-key", + "source", + "--target-key", + "target", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "25", + "--lr-decay-iters", + "1000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.99", + "--adam-eps", + "1e-6", + "--save-interval", + "100", + "--eval-interval", + "100", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--hf-transformer-model-dir", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "SHARD_GRAD_OP", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-sampler-state", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara", + "--num-workers", + "4", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:27:01" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.025999999999, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48781967163086 + } +} diff --git a/wandb/run-20240831_192718-8hc38ffs/files/wandb-summary.json b/wandb/run-20240831_192718-8hc38ffs/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fe9e22cf416d08fa3b43b9f0c278f2270e8d86cd --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 0}} \ No newline at end of file diff --git a/wandb/run-20240831_192718-8hc38ffs/logs/debug-internal.log b/wandb/run-20240831_192718-8hc38ffs/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..76241a5ba2a61152fb895c9b227991ce6aa30c3f --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/logs/debug-internal.log @@ -0,0 +1,185 @@ +2024-08-31 19:27:18,869 INFO StreamThr :14096 [internal.py:wandb_internal():86] W&B internal server running at pid: 14096, started at: 2024-08-31 19:27:18.868088 +2024-08-31 19:27:18,870 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: status +2024-08-31 19:27:18,872 INFO WriterThread:14096 [datastore.py:open_for_write():87] open: /project/wandb/run-20240831_192718-8hc38ffs/run-8hc38ffs.wandb +2024-08-31 19:27:18,873 DEBUG SenderThread:14096 [sender.py:send():382] send: header +2024-08-31 19:27:18,886 DEBUG SenderThread:14096 [sender.py:send():382] send: run +2024-08-31 19:27:19,275 INFO SenderThread:14096 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240831_192718-8hc38ffs/files +2024-08-31 19:27:19,275 INFO SenderThread:14096 [sender.py:_start_run_threads():1136] run started: 8hc38ffs with start time 1725100038.866512 +2024-08-31 19:27:19,280 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: check_version +2024-08-31 19:27:19,281 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: check_version +2024-08-31 19:27:19,350 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: run_start +2024-08-31 19:27:19,356 DEBUG HandlerThread:14096 [system_info.py:__init__():27] System info init +2024-08-31 19:27:19,356 DEBUG HandlerThread:14096 [system_info.py:__init__():42] System info init done +2024-08-31 19:27:19,356 INFO HandlerThread:14096 [system_monitor.py:start():194] Starting system monitor +2024-08-31 19:27:19,356 INFO SystemMonitor:14096 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-31 19:27:19,356 INFO HandlerThread:14096 [system_monitor.py:probe():214] Collecting system info +2024-08-31 19:27:19,357 INFO SystemMonitor:14096 [interfaces.py:start():190] Started cpu monitoring +2024-08-31 19:27:19,357 INFO SystemMonitor:14096 [interfaces.py:start():190] Started disk monitoring +2024-08-31 19:27:19,358 INFO SystemMonitor:14096 [interfaces.py:start():190] Started gpu monitoring +2024-08-31 19:27:19,359 INFO SystemMonitor:14096 [interfaces.py:start():190] Started memory monitoring +2024-08-31 19:27:19,359 INFO SystemMonitor:14096 [interfaces.py:start():190] Started network monitoring +2024-08-31 19:27:19,369 DEBUG HandlerThread:14096 [system_info.py:probe():151] Probing system +2024-08-31 19:27:19,371 DEBUG HandlerThread:14096 [system_info.py:_probe_git():136] Probing git +2024-08-31 19:27:19,384 DEBUG HandlerThread:14096 [system_info.py:_probe_git():144] Probing git done +2024-08-31 19:27:19,384 DEBUG HandlerThread:14096 [system_info.py:probe():199] Probing system done +2024-08-31 19:27:19,384 DEBUG HandlerThread:14096 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-31T10:27:19.369639', 'startedAt': '2024-08-31T10:27:18.853756', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--micro-batch-size', '4', '--valid_micro_batch_size', '1', '--global-batch-size', '16', '--train-iters', '1000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--instruction-train-data-path', '/work/datasets/bin/ichikara/train/data', '--instruction-valid-data-path', '/work/datasets/bin/ichikara/valid/data', '--efficient-instruction-tuning', '--remove-padding-masking', '--source-key', 'source', '--target-key', 'target', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '25', '--lr-decay-iters', '1000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.99', '--adam-eps', '1e-6', '--save-interval', '100', '--eval-interval', '100', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--hf-transformer-model-dir', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', '--fsdp-activation-checkpointing', '--sharding-strategy', 'SHARD_GRAD_OP', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-sampler-state', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', '--num-workers', '4', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:27:01'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.025999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48781967163086}} +2024-08-31 19:27:19,384 INFO HandlerThread:14096 [system_monitor.py:probe():224] Finished collecting system info +2024-08-31 19:27:19,384 INFO HandlerThread:14096 [system_monitor.py:probe():227] Publishing system info +2024-08-31 19:27:19,385 INFO HandlerThread:14096 [system_monitor.py:probe():229] Finished publishing system info +2024-08-31 19:27:19,391 DEBUG SenderThread:14096 [sender.py:send():382] send: files +2024-08-31 19:27:19,391 INFO SenderThread:14096 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-31 19:27:19,403 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: python_packages +2024-08-31 19:27:19,403 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 19:27:19,403 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: python_packages +2024-08-31 19:27:19,404 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 19:27:19,405 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: stop_status +2024-08-31 19:27:19,668 DEBUG SenderThread:14096 [sender.py:send():382] send: telemetry +2024-08-31 19:27:20,010 INFO wandb-upload_0:14096 [upload_job.py:push():131] Uploaded file /tmp/tmprjtmbuspwandb/trmzqykn-wandb-metadata.json +2024-08-31 19:27:20,232 DEBUG SenderThread:14096 [sender.py:send():382] send: exit +2024-08-31 19:27:20,232 INFO SenderThread:14096 [sender.py:send_exit():589] handling exit code: 1 +2024-08-31 19:27:20,232 INFO SenderThread:14096 [sender.py:send_exit():591] handling runtime: 0 +2024-08-31 19:27:20,233 INFO SenderThread:14096 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 19:27:20,233 INFO SenderThread:14096 [sender.py:send_exit():597] send defer +2024-08-31 19:27:20,234 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,234 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-31 19:27:20,234 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,234 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-31 19:27:20,234 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 1 +2024-08-31 19:27:20,234 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,234 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-31 19:27:20,234 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,234 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-31 19:27:20,234 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 2 +2024-08-31 19:27:20,234 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,234 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-31 19:27:20,234 INFO HandlerThread:14096 [system_monitor.py:finish():203] Stopping system monitor +2024-08-31 19:27:20,235 DEBUG SystemMonitor:14096 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-31 19:27:20,235 INFO HandlerThread:14096 [interfaces.py:finish():202] Joined cpu monitor +2024-08-31 19:27:20,235 DEBUG SystemMonitor:14096 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-31 19:27:20,235 INFO HandlerThread:14096 [interfaces.py:finish():202] Joined disk monitor +2024-08-31 19:27:20,235 DEBUG SystemMonitor:14096 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-31 19:27:20,268 INFO HandlerThread:14096 [interfaces.py:finish():202] Joined gpu monitor +2024-08-31 19:27:20,268 INFO HandlerThread:14096 [interfaces.py:finish():202] Joined memory monitor +2024-08-31 19:27:20,268 INFO HandlerThread:14096 [interfaces.py:finish():202] Joined network monitor +2024-08-31 19:27:20,269 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,269 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-31 19:27:20,269 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 3 +2024-08-31 19:27:20,269 DEBUG SenderThread:14096 [sender.py:send():382] send: stats +2024-08-31 19:27:20,269 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,269 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-31 19:27:20,270 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,270 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-31 19:27:20,270 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 4 +2024-08-31 19:27:20,270 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,270 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-31 19:27:20,270 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,270 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-31 19:27:20,270 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 5 +2024-08-31 19:27:20,270 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,270 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-31 19:27:20,270 DEBUG SenderThread:14096 [sender.py:send():382] send: summary +2024-08-31 19:27:20,271 INFO SenderThread:14096 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 19:27:20,272 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,272 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-31 19:27:20,272 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 6 +2024-08-31 19:27:20,272 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,272 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-31 19:27:20,272 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,272 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-31 19:27:20,275 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 19:27:20,277 INFO Thread-12 :14096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192718-8hc38ffs/files/requirements.txt +2024-08-31 19:27:20,277 INFO Thread-12 :14096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192718-8hc38ffs/files/output.log +2024-08-31 19:27:20,277 INFO Thread-12 :14096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192718-8hc38ffs/files/wandb-metadata.json +2024-08-31 19:27:20,277 INFO Thread-12 :14096 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_192718-8hc38ffs/files/wandb-summary.json +2024-08-31 19:27:20,466 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 7 +2024-08-31 19:27:20,466 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:20,466 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-31 19:27:20,466 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:20,466 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-31 19:27:21,232 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:27:21,277 INFO Thread-12 :14096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_192718-8hc38ffs/files/config.yaml +2024-08-31 19:27:22,221 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 8 +2024-08-31 19:27:22,221 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:27:22,221 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:22,222 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-31 19:27:22,222 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:22,222 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-31 19:27:22,222 INFO SenderThread:14096 [job_builder.py:build():296] Attempting to build job artifact +2024-08-31 19:27:22,223 INFO SenderThread:14096 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-31 19:27:22,232 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:27:22,237 INFO SenderThread:14096 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-31 19:27:22,246 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 9 +2024-08-31 19:27:22,246 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:27:22,247 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:22,247 DEBUG SenderThread:14096 [sender.py:send():382] send: artifact +2024-08-31 19:27:22,247 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-31 19:27:22,277 INFO Thread-12 :14096 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_192718-8hc38ffs/files/output.log +2024-08-31 19:27:23,083 INFO SenderThread:14096 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE3NDAxOTIzNw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjgwMzg3NA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE3NDAxOTIzNw==', 'versionIndex': 2}}} +2024-08-31 19:27:23,083 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:23,083 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-31 19:27:23,083 INFO SenderThread:14096 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-31 19:27:23,233 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:27:23,279 INFO SenderThread:14096 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240831_192718-8hc38ffs/files +2024-08-31 19:27:23,279 INFO SenderThread:14096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192718-8hc38ffs/files/requirements.txt requirements.txt +2024-08-31 19:27:23,279 INFO SenderThread:14096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192718-8hc38ffs/files/config.yaml config.yaml +2024-08-31 19:27:23,281 INFO SenderThread:14096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192718-8hc38ffs/files/wandb-metadata.json wandb-metadata.json +2024-08-31 19:27:23,281 INFO SenderThread:14096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192718-8hc38ffs/files/wandb-summary.json wandb-summary.json +2024-08-31 19:27:23,283 INFO SenderThread:14096 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_192718-8hc38ffs/files/output.log output.log +2024-08-31 19:27:23,284 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 10 +2024-08-31 19:27:23,284 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:27:23,286 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:23,286 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-31 19:27:23,286 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:23,286 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-31 19:27:23,286 INFO SenderThread:14096 [file_pusher.py:finish():172] shutting down file pusher +2024-08-31 19:27:23,715 INFO wandb-upload_2:14096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192718-8hc38ffs/files/wandb-summary.json +2024-08-31 19:27:23,720 INFO wandb-upload_1:14096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192718-8hc38ffs/files/config.yaml +2024-08-31 19:27:23,834 INFO wandb-upload_3:14096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192718-8hc38ffs/files/output.log +2024-08-31 19:27:24,115 INFO wandb-upload_0:14096 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_192718-8hc38ffs/files/requirements.txt +2024-08-31 19:27:24,233 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:27:24,234 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:27:24,316 INFO Thread-11 (_thread_body):14096 [sender.py:transition_state():617] send defer: 11 +2024-08-31 19:27:24,317 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:24,317 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-31 19:27:24,317 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:24,317 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-31 19:27:24,317 INFO SenderThread:14096 [file_pusher.py:join():178] waiting for file pusher +2024-08-31 19:27:24,317 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 12 +2024-08-31 19:27:24,317 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:24,317 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-31 19:27:24,318 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:24,318 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-31 19:27:24,318 INFO SenderThread:14096 [file_stream.py:finish():595] file stream finish called +2024-08-31 19:27:24,637 INFO SenderThread:14096 [file_stream.py:finish():599] file stream finish is done +2024-08-31 19:27:24,637 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 13 +2024-08-31 19:27:24,637 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:24,637 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-31 19:27:24,637 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:24,637 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-31 19:27:24,638 INFO SenderThread:14096 [sender.py:transition_state():617] send defer: 14 +2024-08-31 19:27:24,638 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: defer +2024-08-31 19:27:24,638 DEBUG SenderThread:14096 [sender.py:send():382] send: final +2024-08-31 19:27:24,638 INFO HandlerThread:14096 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-31 19:27:24,638 DEBUG SenderThread:14096 [sender.py:send():382] send: footer +2024-08-31 19:27:24,638 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: defer +2024-08-31 19:27:24,638 INFO SenderThread:14096 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-31 19:27:24,639 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:27:24,639 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:27:24,639 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 19:27:24,639 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 19:27:24,640 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: server_info +2024-08-31 19:27:24,640 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: get_summary +2024-08-31 19:27:24,640 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: server_info +2024-08-31 19:27:24,642 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-31 19:27:24,642 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 19:27:24,642 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: job_info +2024-08-31 19:27:24,802 DEBUG SenderThread:14096 [sender.py:send_request():409] send_request: job_info +2024-08-31 19:27:24,802 INFO MainThread:14096 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-31 19:27:24,802 INFO MainThread:14096 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-31 19:27:24,802 INFO MainThread:14096 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-31 19:27:24,803 DEBUG HandlerThread:14096 [handler.py:handle_request():146] handle_request: shutdown +2024-08-31 19:27:24,803 INFO HandlerThread:14096 [handler.py:finish():869] shutting down handler +2024-08-31 19:27:25,642 INFO WriterThread:14096 [datastore.py:close():296] close: /project/wandb/run-20240831_192718-8hc38ffs/run-8hc38ffs.wandb +2024-08-31 19:27:25,802 INFO SenderThread:14096 [sender.py:finish():1572] shutting down sender +2024-08-31 19:27:25,802 INFO SenderThread:14096 [file_pusher.py:finish():172] shutting down file pusher +2024-08-31 19:27:25,802 INFO SenderThread:14096 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240831_192718-8hc38ffs/logs/debug.log b/wandb/run-20240831_192718-8hc38ffs/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..15c4cef1054ddc68307bf777e0f88320e73522f5 --- /dev/null +++ b/wandb/run-20240831_192718-8hc38ffs/logs/debug.log @@ -0,0 +1,28 @@ +2024-08-31 19:27:18,859 INFO MainThread:14025 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_setup.py:_flush():76] Configure stats pid to 14025 +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240831_192718-8hc38ffs/logs/debug.log +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240831_192718-8hc38ffs/logs/debug-internal.log +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_init.py:init():566] calling init triggers +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'SHARD_GRAD_OP', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': None, 'valid_data_path': None, 'test_data_path': None, 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B-3.5e-5-ichikara_train_2024-08-31-19:27:01', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 100, 'save_interval': 100, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 1000, 'lr_warmup_iters': 25, 'min_lr': 1e-06, 'train_iters': 1000, 'train_samples': None, 'global_batch_size': 16, 'micro_batch_size': 4, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.99, 'adam_eps': 1e-06, 'hf_transformer_model_dir': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'instruction_train_data_path': '/work/datasets/bin/ichikara/train/data', 'instruction_valid_data_path': '/work/datasets/bin/ichikara/valid/data', 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': True, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B-3.5e-5-ichikara', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': 'source', 'target_key': 'target', 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': True, 'remove_padding_masking': True, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 4} +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_init.py:init():616] starting backend +2024-08-31 19:27:18,860 INFO MainThread:14025 [wandb_init.py:init():620] setting up manager +2024-08-31 19:27:18,865 INFO MainThread:14025 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-31 19:27:18,866 INFO MainThread:14025 [wandb_init.py:init():628] backend started and connected +2024-08-31 19:27:18,871 INFO MainThread:14025 [wandb_init.py:init():720] updated telemetry +2024-08-31 19:27:18,882 INFO MainThread:14025 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-31 19:27:19,280 INFO MainThread:14025 [wandb_run.py:_on_init():2262] communicating current version +2024-08-31 19:27:19,304 INFO MainThread:14025 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.8 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-31 19:27:19,304 INFO MainThread:14025 [wandb_init.py:init():804] starting run threads in backend +2024-08-31 19:27:19,402 INFO MainThread:14025 [wandb_run.py:_console_start():2241] atexit reg +2024-08-31 19:27:19,402 INFO MainThread:14025 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-31 19:27:19,402 INFO MainThread:14025 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-31 19:27:19,402 INFO MainThread:14025 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-31 19:27:19,403 INFO MainThread:14025 [wandb_init.py:init():847] run started, returning control to user process +2024-08-31 19:27:25,803 WARNING MsgRouterThr:14025 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240831_192718-8hc38ffs/run-8hc38ffs.wandb b/wandb/run-20240831_192718-8hc38ffs/run-8hc38ffs.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5201a7f984160d6d696e596d37e0a87d6f499e07 Binary files /dev/null and b/wandb/run-20240831_192718-8hc38ffs/run-8hc38ffs.wandb differ diff --git a/wandb/run-20240831_204827-vaha0m5b/files/config.yaml b/wandb/run-20240831_204827-vaha0m5b/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07649f35564dc2d92dfe6b308e351f94f98d975d --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/files/config.yaml @@ -0,0 +1,334 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: SHARD_GRAD_OP +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: null +valid_data_path: + desc: null + value: null +test_data_path: + desc: null + value: null +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara_train_2024-08-31-20:48:03 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 100 +save_interval: + desc: null + value: 100 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 1000 +lr_warmup_iters: + desc: null + value: 25 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 1000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 16 +micro_batch_size: + desc: null + value: 4 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.99 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +instruction_train_data_path: + desc: null + value: /work/datasets/bin/ichikara/train/data +instruction_valid_data_path: + desc: null + value: /work/datasets/bin/ichikara/valid/data +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: true +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: source +target_key: + desc: null + value: target +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: true +remove_padding_masking: + desc: null + value: true +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 4 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1725104907.858014 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240831_204827-vaha0m5b/files/output.log b/wandb/run-20240831_204827-vaha0m5b/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ac9ba42a68383f5b574a8445df05df253bfd22ea --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/files/output.log @@ -0,0 +1,3245 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.SHARD_GRAD_OP since the world size is 1. + warnings.warn( +--> applying fsdp activation checkpointing... +Instruction dataset size: 398 +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 24.41711543285516, Tokens per sec: 8061.658934554785, Loss: 2.7056679725646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 51.05587885859487, Tokens per sec: 16203.832266011537, Loss: 2.499762535095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 53.11334010796102, Tokens per sec: 17519.70769213488, Loss: 2.6662216186523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 54.8497719826819, Tokens per sec: 18169.807953865966, Loss: 2.6262600421905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 55.024110558302, Tokens per sec: 17873.102897340446, Loss: 2.682037353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 52.52394220805646, Tokens per sec: 17009.66345811302, Loss: 2.6604278087615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 53.524744728492585, Tokens per sec: 17629.900907400322, Loss: 2.7069907188415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 48.35460108910016, Tokens per sec: 15311.404488263643, Loss: 2.6216440200805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 53.40017476000617, Tokens per sec: 17484.840980621833, Loss: 2.64982271194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 52.48359486011286, Tokens per sec: 17148.501743700475, Loss: 2.7571539878845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 33.73891862606304, Tokens per sec: 11072.407406917599, Loss: 2.5319983959198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 50.42314466168581, Tokens per sec: 16042.433811686275, Loss: 2.7258362770080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 55.956294406945084, Tokens per sec: 18090.980189336766, Loss: 2.5666298866271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 51.6579519663236, Tokens per sec: 16667.928383466387, Loss: 2.5924415588378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 52.886819647384954, Tokens per sec: 17474.737209409912, Loss: 2.578636407852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 48.738925092018704, Tokens per sec: 16111.068190954995, Loss: 2.7193493843078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 54.61903839107271, Tokens per sec: 17575.061180783, Loss: 2.685131549835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 44.28820480605995, Tokens per sec: 13719.977973880434, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 54.390147330148096, Tokens per sec: 17757.973814103356, Loss: 2.9614834785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 47.86904365441833, Tokens per sec: 14995.910897862364, Loss: 2.9522876739501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 51.74432592719105, Tokens per sec: 16782.43239353052, Loss: 2.9492745399475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 52.50546682508966, Tokens per sec: 17403.635696220717, Loss: 2.6783835887908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 54.15069808781063, Tokens per sec: 17915.257711120677, Loss: 2.8714540004730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 53.72331966888494, Tokens per sec: 17871.510926487194, Loss: 2.7480897903442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 54.35701512827702, Tokens per sec: 17856.982256461193, Loss: 2.7533650398254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 56.060219632838056, Tokens per sec: 17887.30279681314, Loss: 3.0266213417053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 43.275318425786175, Tokens per sec: 13870.799691141754, Loss: 3.037449359893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 56.146000502695394, Tokens per sec: 18143.231262801986, Loss: 2.7366185188293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 54.16304378056924, Tokens per sec: 17140.834163569325, Loss: 2.756969928741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 53.1346005492198, Tokens per sec: 17531.197243095357, Loss: 2.638162136077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 54.40491094942427, Tokens per sec: 18081.174627126544, Loss: 2.7234549522399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 55.12996720815161, Tokens per sec: 17752.73213072603, Loss: 2.7919673919677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 53.31138581833595, Tokens per sec: 17361.88375673264, Loss: 3.019876480102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 56.74304713721965, Tokens per sec: 18354.529437113557, Loss: 2.9944465160369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 48.74988340357773, Tokens per sec: 16042.165552414912, Loss: 2.7435216903686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 49.936383083354194, Tokens per sec: 15896.703406104314, Loss: 2.7820472717285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 54.44238713765347, Tokens per sec: 18053.374545872575, Loss: 2.8420777320861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 53.90587650342442, Tokens per sec: 17732.82937357104, Loss: 2.8940823078155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 49.24922168525881, Tokens per sec: 15884.116014461202, Loss: 2.8762614727020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 52.28979138481473, Tokens per sec: 17082.29777568358, Loss: 2.6667847633361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 51.03153851758984, Tokens per sec: 16880.43035229364, Loss: 2.798936367034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 54.42830023084433, Tokens per sec: 17758.461142533703, Loss: 2.764091968536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 54.69525062535499, Tokens per sec: 17839.55180088522, Loss: 2.732822895050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 54.976854667922545, Tokens per sec: 17934.41946552607, Loss: 2.7764174938201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 47.373807029667844, Tokens per sec: 15714.776017582515, Loss: 2.6468663215637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 54.64154044564981, Tokens per sec: 17823.533671579044, Loss: 2.81516170501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 55.5320969693362, Tokens per sec: 18305.08224766974, Loss: 2.7747631072998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 55.92277740317883, Tokens per sec: 18160.444036908695, Loss: 2.63429594039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 52.69951180804521, Tokens per sec: 17207.44377797532, Loss: 2.7244694232940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 52.25662635157453, Tokens per sec: 17055.647466464634, Loss: 2.7651660442352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 53.05883776692659, Tokens per sec: 17380.419468102664, Loss: 2.57653546333313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 53.88748136501899, Tokens per sec: 17799.302944016592, Loss: 2.5505576133728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 50.052844601687944, Tokens per sec: 16289.725104822655, Loss: 2.9826557636260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 50.572921823889835, Tokens per sec: 16243.448029196341, Loss: 2.929823875427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 55.06689370444475, Tokens per sec: 18094.788346673813, Loss: 2.5789990425109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 53.46347741637723, Tokens per sec: 17234.713407787905, Loss: 2.7921276092529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 50.967147054051026, Tokens per sec: 16520.688297768316, Loss: 2.78560209274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 51.76579242784537, Tokens per sec: 16931.097290429218, Loss: 2.746201992034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 48.51851702385823, Tokens per sec: 15745.418169088414, Loss: 2.5430994033813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 54.07615521319375, Tokens per sec: 18024.471446655898, Loss: 2.6407463550567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 51.893633129332, Tokens per sec: 16596.118410614497, Loss: 2.72259521484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 56.21396105763064, Tokens per sec: 18065.76705345787, Loss: 2.8725836277008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 51.69013766178614, Tokens per sec: 16783.117640678083, Loss: 2.7218985557556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 46.68167309200986, Tokens per sec: 15190.024157028993, Loss: 2.5542826652526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 54.38266754162839, Tokens per sec: 18168.830779519856, Loss: 2.664468765258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 55.73529087800717, Tokens per sec: 18428.49253910037, Loss: 2.780092239379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 55.15397010682973, Tokens per sec: 18034.70607166661, Loss: 2.7988710403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 55.241527613531886, Tokens per sec: 18104.58266384015, Loss: 2.8049190044403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 51.35027434795188, Tokens per sec: 16303.94216583752, Loss: 2.8518552780151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 53.56465934109697, Tokens per sec: 17607.13129092683, Loss: 2.68117094039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 50.32589608404978, Tokens per sec: 16247.597524027675, Loss: 2.5667238235473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 50.46275529169757, Tokens per sec: 16205.377519443944, Loss: 2.8021979331970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 52.69093152534077, Tokens per sec: 17264.282834796046, Loss: 2.6207804679870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 54.21446130018664, Tokens per sec: 18048.83739742614, Loss: 2.5788493156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 49.58204580604748, Tokens per sec: 15854.232188196036, Loss: 2.6871094703674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 55.4509986337839, Tokens per sec: 17708.955781151144, Loss: 2.7905426025390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 54.21391700554822, Tokens per sec: 17620.27181787434, Loss: 2.715909481048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 53.688287988840536, Tokens per sec: 17691.28693951759, Loss: 2.755496025085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 56.79036633447602, Tokens per sec: 18352.98447142769, Loss: 2.6972286701202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 53.54623466568345, Tokens per sec: 17429.578310476598, Loss: 2.8700037002563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 56.01200740371439, Tokens per sec: 18112.014621003163, Loss: 2.7348930835723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 47.956818178624644, Tokens per sec: 15602.330930847216, Loss: 2.6523852348327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 54.17876261390912, Tokens per sec: 17555.839984559323, Loss: 2.847149133682251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 52.88639346188313, Tokens per sec: 17278.655406630136, Loss: 2.6320271492004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 52.94424609886651, Tokens per sec: 17630.506815409466, Loss: 2.545382022857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 53.124075072493696, Tokens per sec: 17222.7244718157, Loss: 2.773731231689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 54.41836049540014, Tokens per sec: 18071.69591552176, Loss: 2.5173473358154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 56.380255852679404, Tokens per sec: 18523.213244857478, Loss: 2.723914861679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 55.399675611778335, Tokens per sec: 17710.079164727864, Loss: 2.740046977996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 54.28125273170149, Tokens per sec: 17581.699797812493, Loss: 2.665520429611206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 48.82404262070625, Tokens per sec: 15248.242387599043, Loss: 2.93502140045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 51.66488403228486, Tokens per sec: 16958.206556064943, Loss: 2.807143449783325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 57.335335704561246, Tokens per sec: 18521.379804538687, Loss: 2.640833616256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 53.37000052497022, Tokens per sec: 17613.363781321088, Loss: 2.6946539878845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 56.29711360878284, Tokens per sec: 18582.54545400302, Loss: 2.685352325439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 55.78436225080444, Tokens per sec: 18120.050218980035, Loss: 2.71795654296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 55.015466580014234, Tokens per sec: 17685.000639262988, Loss: 2.679778575897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 55.409779639774364, Tokens per sec: 18411.979187326437, Loss: 2.5902199745178223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 55.832043584596285, Tokens per sec: 17807.167111191124, Loss: 2.6823878288269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 50.56164236508224, Tokens per sec: 16487.159923812807, Loss: 2.642475128173828 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/optimizer.pt +[rank0]:[2024-08-31 20:50:24,541] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00695493599960173, 'preprocessing_with_comm': 0.0008779729996604146, 'state_converting': 0.9717823969986057, : 0.9811227950012835}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000100, took 4.79s + eval ppl=14.378462791442871, eval loss=2.665731430053711 +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 47.15879060902562, Tokens per sec: 15372.382706176719, Loss: 2.638658046722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 54.54085322143742, Tokens per sec: 18075.199535096646, Loss: 2.6302032470703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 54.51578289557156, Tokens per sec: 17839.573803008512, Loss: 2.6312789916992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 54.326598351073855, Tokens per sec: 17578.756085818914, Loss: 2.6221156120300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 52.9894904343226, Tokens per sec: 17615.29946823723, Loss: 2.688459873199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 55.20718507594677, Tokens per sec: 18052.106746990252, Loss: 2.688093900680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 53.35289957506106, Tokens per sec: 17631.744923334456, Loss: 2.6480722427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 54.8505277485906, Tokens per sec: 17825.68407687196, Loss: 2.714466094970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 52.82201376767877, Tokens per sec: 17126.229526810464, Loss: 2.731403112411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 50.11119811986029, Tokens per sec: 16580.242003045925, Loss: 2.5759329795837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 52.97630932442042, Tokens per sec: 17232.48496903207, Loss: 2.5841915607452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 56.57001714735485, Tokens per sec: 18582.406762109178, Loss: 2.708066940307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 54.782103578967096, Tokens per sec: 17922.183549055237, Loss: 2.6548988819122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 53.305621937576355, Tokens per sec: 17179.539182918244, Loss: 2.558598041534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 39.17414263211029, Tokens per sec: 12649.38772152638, Loss: 2.577883243560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 53.06802718233328, Tokens per sec: 17482.43981677237, Loss: 2.6572389602661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 59.90704052497011, Tokens per sec: 18558.51417058315, Loss: 2.6708009243011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 55.01681393602537, Tokens per sec: 18037.03779181951, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 45.78677921684916, Tokens per sec: 14686.717137731259, Loss: 2.6592774391174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 55.439626577710065, Tokens per sec: 17833.18324077349, Loss: 2.5787911415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 56.07747520831232, Tokens per sec: 18481.7115126447, Loss: 2.6806375980377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 48.54646662783645, Tokens per sec: 15282.250560003964, Loss: 2.554830312728882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 52.870433224334064, Tokens per sec: 17049.210150694613, Loss: 2.5314927101135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 49.705988403004824, Tokens per sec: 16220.431673700521, Loss: 2.473952293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 53.28969917085387, Tokens per sec: 17261.978348703375, Loss: 2.6064672470092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 54.96652472572785, Tokens per sec: 18210.040683017094, Loss: 2.803375482559204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 56.50925023711918, Tokens per sec: 18633.521841892496, Loss: 2.676948308944702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 52.61922252983411, Tokens per sec: 17481.713860388107, Loss: 2.508626699447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 49.07746276724136, Tokens per sec: 16285.486531711194, Loss: 2.5916929244995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 53.14030516793038, Tokens per sec: 17226.545088105468, Loss: 2.4968090057373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 55.65471121739625, Tokens per sec: 18480.689716306344, Loss: 2.5129170417785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 53.43097305004482, Tokens per sec: 17402.2921108013, Loss: 2.7639598846435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 54.713790553563705, Tokens per sec: 17811.119896137778, Loss: 2.732116222381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 56.673009267805334, Tokens per sec: 18522.036999770804, Loss: 2.74287486076355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 52.877747622161095, Tokens per sec: 17360.72182066765, Loss: 2.4602649211883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 57.5221292077773, Tokens per sec: 18742.644373421826, Loss: 2.785938262939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 51.53365119963602, Tokens per sec: 16616.753257141718, Loss: 2.6054718494415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 45.704361626461235, Tokens per sec: 14850.812211885748, Loss: 2.6490464210510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 53.18630220469384, Tokens per sec: 17630.8156673915, Loss: 2.521334648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 54.379238785811204, Tokens per sec: 17757.40440503903, Loss: 2.5881125926971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 55.488435052285574, Tokens per sec: 18203.977298326754, Loss: 2.689542293548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 52.81732173932099, Tokens per sec: 17405.767442091143, Loss: 2.5242433547973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 52.53028761566891, Tokens per sec: 17031.644548277305, Loss: 2.5958597660064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 53.74226953776598, Tokens per sec: 17537.581266047866, Loss: 2.5727696418762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 55.09167025255932, Tokens per sec: 17899.548865688976, Loss: 2.574427604675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 52.025926146379355, Tokens per sec: 16783.8638404456, Loss: 2.6001381874084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 56.24887690605393, Tokens per sec: 18433.167377263868, Loss: 2.4470057487487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 49.576703840205596, Tokens per sec: 16245.273811856101, Loss: 2.600644111633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 55.22567407613089, Tokens per sec: 18417.12950478553, Loss: 2.544185161590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 54.25428525889399, Tokens per sec: 18046.591911716765, Loss: 2.539757251739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 53.61093873444605, Tokens per sec: 17736.683320411372, Loss: 2.448965549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 51.31320875810534, Tokens per sec: 16453.9450376152, Loss: 2.697970151901245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 54.97190958712155, Tokens per sec: 17698.893646147244, Loss: 2.780695915222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 52.949367339574486, Tokens per sec: 17381.290720952464, Loss: 2.6103057861328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 50.91500221499501, Tokens per sec: 16594.005885326707, Loss: 2.445490837097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 57.396880535637244, Tokens per sec: 18631.340003883462, Loss: 2.642939567565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 46.8526722767666, Tokens per sec: 15436.199615628, Loss: 2.6216607093811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 55.57236747392144, Tokens per sec: 18118.0100499745, Loss: 2.541896343231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 57.70301804947574, Tokens per sec: 18651.037550554578, Loss: 2.5098934173583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 49.00164327482849, Tokens per sec: 16112.698343304935, Loss: 2.3881778717041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 52.30448198324661, Tokens per sec: 16773.191738077592, Loss: 2.74949312210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 51.793971012734154, Tokens per sec: 16866.290865909854, Loss: 2.5180225372314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 53.692172621689224, Tokens per sec: 17737.815244248188, Loss: 2.5802128314971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 53.58773414886049, Tokens per sec: 17815.70194476556, Loss: 2.5312845706939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 56.62823789009672, Tokens per sec: 17613.178863982754, Loss: 2.610612154006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 49.85717272792569, Tokens per sec: 16196.119623136157, Loss: 2.567363739013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 55.149576375210735, Tokens per sec: 18079.033929416262, Loss: 2.5444278717041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 54.00028564447943, Tokens per sec: 17374.519121053752, Loss: 2.804875612258911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 59.587624938561845, Tokens per sec: 18459.562884812454, Loss: 2.6698169708251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 53.090078239707765, Tokens per sec: 17612.477487812812, Loss: 2.3850884437561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 51.89608605393986, Tokens per sec: 16957.96878791155, Loss: 2.6229147911071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 54.51297171308156, Tokens per sec: 17943.200917649592, Loss: 2.5217514038085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 54.78896506991529, Tokens per sec: 18155.872774186235, Loss: 2.5200061798095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 55.78444208169312, Tokens per sec: 18493.65933275332, Loss: 2.3435399532318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 49.19307104403389, Tokens per sec: 16171.527014683388, Loss: 2.6218204498291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 54.53920875372551, Tokens per sec: 17656.397246817887, Loss: 2.672213315963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 51.386818684645434, Tokens per sec: 16662.297134185752, Loss: 2.65544056892395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 54.64117028677778, Tokens per sec: 17885.135993480646, Loss: 2.636399745941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 51.40919157638255, Tokens per sec: 16473.817611963525, Loss: 2.6187515258789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 50.81119510245098, Tokens per sec: 16069.585120096019, Loss: 2.6521503925323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 53.056833082505705, Tokens per sec: 17481.724333818674, Loss: 2.5049633979797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 54.7933537059422, Tokens per sec: 17648.7367899358, Loss: 2.6025097370147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 51.96272942324709, Tokens per sec: 16524.1224401749, Loss: 2.6212868690490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 55.203084469210765, Tokens per sec: 18249.407986980368, Loss: 2.4497315883636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 45.74561693964139, Tokens per sec: 14933.083445855478, Loss: 2.553154230117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 54.19754477534652, Tokens per sec: 17929.22583637002, Loss: 2.500713348388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 50.98585165525692, Tokens per sec: 16193.54336667826, Loss: 2.5688610076904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 56.28238555136044, Tokens per sec: 18268.03866239123, Loss: 2.507455587387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 55.249956688454176, Tokens per sec: 17874.54724090737, Loss: 2.5920209884643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 54.63227733353676, Tokens per sec: 17870.150048343876, Loss: 2.61616849899292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 54.68499852358584, Tokens per sec: 17837.709269651088, Loss: 2.8484668731689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 53.26241913541719, Tokens per sec: 17689.33730111034, Loss: 2.535823345184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 53.769171648825875, Tokens per sec: 17324.61136416341, Loss: 2.5795414447784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 48.887099872353346, Tokens per sec: 16090.052100511562, Loss: 2.5195236206054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 53.92790529398391, Tokens per sec: 17887.361596921462, Loss: 2.6277925968170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 53.05598222302013, Tokens per sec: 17563.577252195628, Loss: 2.459408760070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 52.02974729503185, Tokens per sec: 16495.351102109074, Loss: 2.6883554458618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 53.55328493792837, Tokens per sec: 17757.024231292216, Loss: 2.3827996253967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 50.455983216211585, Tokens per sec: 16698.597886091964, Loss: 2.5726640224456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 48.954872088563285, Tokens per sec: 16163.149289604866, Loss: 2.5752170085906982 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/model.pt +[rank0]:[2024-08-31 20:52:00,449] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006622572998821852, 'preprocessing_with_comm': 0.0007930350002425257, 'state_converting': 0.9857889770009933, : 0.9946575169997232}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000200, took 4.36s + eval ppl=13.641901016235352, eval loss=2.6131460666656494 +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 50.89128257628959, Tokens per sec: 16151.593608883122, Loss: 2.472086191177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 52.973612919629936, Tokens per sec: 17473.61796229999, Loss: 2.4649219512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 45.39854784545822, Tokens per sec: 15038.948493656773, Loss: 2.4763944149017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 54.48753004125316, Tokens per sec: 17698.809143264032, Loss: 2.609239339828491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 52.59048537571925, Tokens per sec: 17031.213190288436, Loss: 2.5244545936584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 52.919529054733246, Tokens per sec: 17430.55586782832, Loss: 2.5881268978118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 54.7602428187869, Tokens per sec: 17339.740351264754, Loss: 2.57930588722229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 53.71446517153577, Tokens per sec: 17587.749218132787, Loss: 2.5826632976531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 52.967896892286866, Tokens per sec: 17306.74270004788, Loss: 2.6186604499816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 53.17105860932056, Tokens per sec: 17449.629305092792, Loss: 2.356133222579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 50.81419858599246, Tokens per sec: 16166.85002451039, Loss: 2.7463128566741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 50.69391873135736, Tokens per sec: 16389.624960136865, Loss: 2.456305503845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 52.181197055219954, Tokens per sec: 16836.7583764636, Loss: 2.4515810012817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 53.591272665417705, Tokens per sec: 17707.50090079037, Loss: 2.3824729919433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 42.593646964798886, Tokens per sec: 14079.693991111679, Loss: 2.5503296852111816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 54.61802490156068, Tokens per sec: 17574.735064822253, Loss: 2.624717950820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 46.22391628316212, Tokens per sec: 14319.63919171321, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 55.074247595720514, Tokens per sec: 17981.327402916348, Loss: 2.5647597312927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 52.75577143056492, Tokens per sec: 16526.773616621518, Loss: 2.516895294189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 52.06428700145874, Tokens per sec: 16886.20657555416, Loss: 2.563632011413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 52.646906928185466, Tokens per sec: 17450.517900607465, Loss: 2.3624470233917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 47.0390017802397, Tokens per sec: 15562.418752207272, Loss: 2.4653537273406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 54.633535758791794, Tokens per sec: 18174.30190992778, Loss: 2.4495487213134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 55.220921254639606, Tokens per sec: 18140.786588492654, Loss: 2.467264413833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 56.55678382462326, Tokens per sec: 18045.743026171676, Loss: 2.6465747356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 52.967460343705504, Tokens per sec: 16977.368608759953, Loss: 2.682344436645508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 51.85655361118743, Tokens per sec: 16757.12314743587, Loss: 2.4245076179504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 54.63665558416103, Tokens per sec: 17290.7168292512, Loss: 2.4448206424713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 53.63857219095338, Tokens per sec: 17697.477334877993, Loss: 2.3742752075195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 54.84333302305934, Tokens per sec: 18226.88180567798, Loss: 2.42581844329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 48.98863177491586, Tokens per sec: 15775.123791879603, Loss: 2.4993579387664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 54.33443265898541, Tokens per sec: 17695.05874463437, Loss: 2.697145938873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 57.8319408167357, Tokens per sec: 18706.751111889545, Loss: 2.6491451263427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 55.73470075228261, Tokens per sec: 18340.66533206936, Loss: 2.4990715980529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 50.35053886300988, Tokens per sec: 16028.545385570886, Loss: 2.4910449981689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 54.39038496792128, Tokens per sec: 18036.130359922397, Loss: 2.5592355728149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 53.319614255879884, Tokens per sec: 17539.973064051366, Loss: 2.596277952194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 51.65687335749097, Tokens per sec: 16660.644397560383, Loss: 2.4742581844329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 52.912110315489954, Tokens per sec: 17285.60012215893, Loss: 2.416299819946289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 55.58659781743673, Tokens per sec: 18387.172329808775, Loss: 2.5172975063323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 48.600478593886315, Tokens per sec: 15857.002826793636, Loss: 2.4915637969970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 55.02209100340778, Tokens per sec: 17946.154948109684, Loss: 2.467926263809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 55.01886369198132, Tokens per sec: 17948.123549969576, Loss: 2.50532603263855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 53.42343300420855, Tokens per sec: 17721.54986880625, Loss: 2.3984875679016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 54.613208211239396, Tokens per sec: 17814.29197506232, Loss: 2.5416383743286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 55.34383899919902, Tokens per sec: 18243.026647121405, Loss: 2.5150225162506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 55.84314933563377, Tokens per sec: 18134.585502487316, Loss: 2.3693349361419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 52.61101053906154, Tokens per sec: 17178.546344999795, Loss: 2.4669408798217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 48.8987896925698, Tokens per sec: 15959.708399892448, Loss: 2.510443925857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 52.78137316767174, Tokens per sec: 17289.530724105254, Loss: 2.3515069484710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 53.96267703563872, Tokens per sec: 17824.140447784357, Loss: 2.3435139656066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 50.83496677688538, Tokens per sec: 16544.267185971876, Loss: 2.731174945831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 56.979836345406575, Tokens per sec: 18301.276197008458, Loss: 2.664489507675171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 54.41215984982402, Tokens per sec: 17879.645095878685, Loss: 2.3527584075927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 53.49754644378771, Tokens per sec: 17245.69604400756, Loss: 2.5239620208740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 52.21665110958055, Tokens per sec: 16925.707378124316, Loss: 2.5404958724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 56.11119944405181, Tokens per sec: 18352.35456298919, Loss: 2.4800493717193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 50.64328862421782, Tokens per sec: 16434.957326788015, Loss: 2.300445079803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 48.990236839669485, Tokens per sec: 16329.251249469165, Loss: 2.4058165550231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 54.340846365367824, Tokens per sec: 17378.762411277406, Loss: 2.4663000106811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 57.33406531189756, Tokens per sec: 18425.740664149132, Loss: 2.6021366119384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 57.248295880948874, Tokens per sec: 18587.779564158955, Loss: 2.493281602859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 52.810055777262704, Tokens per sec: 17184.174641931866, Loss: 2.3436944484710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 53.97600223506708, Tokens per sec: 18032.967029674222, Loss: 2.420156717300415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 55.23122811250093, Tokens per sec: 18261.827634924986, Loss: 2.533273696899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 55.08075747029156, Tokens per sec: 18010.766391927507, Loss: 2.566554307937622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 54.835494379643684, Tokens per sec: 17971.51135743778, Loss: 2.570080518722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 44.99948283222696, Tokens per sec: 14287.537406671894, Loss: 2.6095268726348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 53.24083165982079, Tokens per sec: 17500.68654601474, Loss: 2.4512600898742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 50.98870956985676, Tokens per sec: 16461.585303458447, Loss: 2.3492867946624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 56.425802530933495, Tokens per sec: 18120.323128725704, Loss: 2.5651473999023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 52.40976911897095, Tokens per sec: 17172.159443435125, Loss: 2.4142637252807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 54.582853444010624, Tokens per sec: 18171.48087193292, Loss: 2.353193521499634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 52.41887059562494, Tokens per sec: 16761.328258155005, Loss: 2.466569423675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 55.524725558638465, Tokens per sec: 17732.50137788156, Loss: 2.5467751026153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 53.18234941033404, Tokens per sec: 17284.998101637888, Loss: 2.4856345653533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 53.6553726705334, Tokens per sec: 17680.440731476872, Loss: 2.524536609649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 50.2107866921758, Tokens per sec: 16226.6568775458, Loss: 2.4721474647521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 53.28824411799428, Tokens per sec: 17345.60104330952, Loss: 2.6571879386901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 56.052762073804786, Tokens per sec: 18125.193030682913, Loss: 2.5017905235290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 51.575287587910566, Tokens per sec: 16779.568273336088, Loss: 2.441197633743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 54.31352324889603, Tokens per sec: 17599.507208207586, Loss: 2.61430287361145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 52.50395717901564, Tokens per sec: 17153.708623268503, Loss: 2.4020614624023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 52.53434123476922, Tokens per sec: 17494.007931533746, Loss: 2.342539072036743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 53.38610008799091, Tokens per sec: 17307.67248531944, Loss: 2.5620498657226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 54.55611269922438, Tokens per sec: 18117.441798281587, Loss: 2.328301429748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 45.42571381919052, Tokens per sec: 14924.19945860794, Loss: 2.4959969520568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 55.35158400889008, Tokens per sec: 17694.70531849313, Loss: 2.5144948959350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 54.10844283995788, Tokens per sec: 17525.72666738851, Loss: 2.4520013332366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 54.56952468166298, Tokens per sec: 17042.614553371233, Loss: 2.713758945465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 52.33449145939886, Tokens per sec: 17177.9949340545, Loss: 2.584197998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 56.79260873944053, Tokens per sec: 18346.059434863662, Loss: 2.410907030105591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 53.02051363744583, Tokens per sec: 17498.02483385586, Loss: 2.478766441345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 56.17967441730535, Tokens per sec: 18543.781137791768, Loss: 2.4910264015197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 55.68548818089842, Tokens per sec: 18087.933635769903, Loss: 2.509826183319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 54.81117677403973, Tokens per sec: 17619.33064546225, Loss: 2.4771862030029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 48.87461081156099, Tokens per sec: 16240.424035276039, Loss: 2.358344078063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 55.81896399754241, Tokens per sec: 17802.995485410294, Loss: 2.4891295433044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 49.87253945768953, Tokens per sec: 16262.456980895862, Loss: 2.446200370788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 55.909873639661036, Tokens per sec: 18224.979129943145, Loss: 2.4317376613616943 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/model.pt +[rank0]:[2024-08-31 20:53:35,792] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00671758799944655, 'preprocessing_with_comm': 0.0007811930008756462, 'state_converting': 1.0154695019991777, : 1.0257078670001647}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000300, took 4.49s + eval ppl=13.395930290222168, eval loss=2.5949509143829346 +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 53.03699119124127, Tokens per sec: 17576.809710524434, Loss: 2.4198508262634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 54.16964742650213, Tokens per sec: 17726.305517783087, Loss: 2.4387927055358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 54.11966700563081, Tokens per sec: 17511.79817278069, Loss: 2.4176807403564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 52.48344802006567, Tokens per sec: 17447.0757582581, Loss: 2.4890494346618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 54.945403943205804, Tokens per sec: 17966.507364480763, Loss: 2.497939109802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 44.89738219531729, Tokens per sec: 14837.416464677433, Loss: 2.4525177478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 54.85272480051661, Tokens per sec: 17826.39808920951, Loss: 2.5023131370544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 52.65069692586183, Tokens per sec: 17070.684284486506, Loss: 2.540153980255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 55.624574957059984, Tokens per sec: 18404.447482949017, Loss: 2.3985884189605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 52.816157254010726, Tokens per sec: 17180.389642247534, Loss: 2.396353244781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 56.11549051273134, Tokens per sec: 18433.101542936525, Loss: 2.5125865936279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 54.76318284436536, Tokens per sec: 17915.993555311634, Loss: 2.45831036567688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 53.08736597487636, Tokens per sec: 17109.198818678593, Loss: 2.368788242340088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 35.224763876393524, Tokens per sec: 11374.127568180438, Loss: 2.3961520195007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 53.14481838999429, Tokens per sec: 17507.737491053314, Loss: 2.4734153747558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 59.55468372291658, Tokens per sec: 18449.358073958978, Loss: 2.4412965774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 57.140877420668986, Tokens per sec: 18733.40333180317, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 52.15061171931226, Tokens per sec: 16728.000876710263, Loss: 2.4648141860961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 55.766136821701785, Tokens per sec: 17938.211311316725, Loss: 2.378394365310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 55.720725442030904, Tokens per sec: 18364.135850792518, Loss: 2.481067657470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 46.69946457031117, Tokens per sec: 14700.821051563502, Loss: 2.366185188293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 52.92680820683857, Tokens per sec: 17067.3894782572, Loss: 2.3555943965911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 53.28806669360209, Tokens per sec: 17389.36238062851, Loss: 2.311187267303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 53.1725719867652, Tokens per sec: 17224.037678231565, Loss: 2.4241151809692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 48.17569261337648, Tokens per sec: 15960.283587139136, Loss: 2.6109378337860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 56.22551056694649, Tokens per sec: 18539.960711291944, Loss: 2.487144947052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 52.63286381934439, Tokens per sec: 17486.245913665214, Loss: 2.335569143295288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 54.97947824916586, Tokens per sec: 18243.965805521362, Loss: 2.4127185344696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 52.7986040132725, Tokens per sec: 17115.77548810468, Loss: 2.3253021240234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 54.72015263744854, Tokens per sec: 18170.360424141643, Loss: 2.3342788219451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 52.97603645064068, Tokens per sec: 17254.12075731855, Loss: 2.5901906490325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 54.680496521971655, Tokens per sec: 17800.28160505043, Loss: 2.5506277084350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 49.78897337882947, Tokens per sec: 16272.176456088779, Loss: 2.5650887489318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 52.35458242628933, Tokens per sec: 17188.957215702176, Loss: 2.301792621612549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 57.57793369145714, Tokens per sec: 18760.82735111161, Loss: 2.594849109649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 52.64936985038485, Tokens per sec: 16976.510834808087, Loss: 2.432803153991699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 55.39539117191015, Tokens per sec: 17999.738371177595, Loss: 2.4321072101593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 52.96647516083767, Tokens per sec: 17557.944835461512, Loss: 2.332202196121216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 53.700547648070035, Tokens per sec: 17535.77951164073, Loss: 2.420654773712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 50.828076207087655, Tokens per sec: 16675.062912110978, Loss: 2.513399124145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 52.44797903336305, Tokens per sec: 17284.05181860562, Loss: 2.3622398376464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 52.14877179462235, Tokens per sec: 16907.947493710428, Loss: 2.4123058319091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 47.76641477183161, Tokens per sec: 15587.495430576655, Loss: 2.4040045738220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 54.653956994518445, Tokens per sec: 17757.33372108463, Loss: 2.4091200828552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 52.47973535933552, Tokens per sec: 16930.265309943054, Loss: 2.431924343109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 56.38204603092546, Tokens per sec: 18476.80787114147, Loss: 2.274045467376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 55.53858748760918, Tokens per sec: 18198.86137989352, Loss: 2.4384753704071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 54.893716443608305, Tokens per sec: 18306.425438053044, Loss: 2.388054847717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 54.17044114388314, Tokens per sec: 18018.702860730664, Loss: 2.3749001026153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 53.63228929446896, Tokens per sec: 17743.746955758543, Loss: 2.2925939559936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 51.30616072003889, Tokens per sec: 16451.68503412341, Loss: 2.542083501815796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 48.82860048276908, Tokens per sec: 15720.978465649041, Loss: 2.6212286949157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 52.785146307630974, Tokens per sec: 17327.38311747914, Loss: 2.4476563930511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 55.21048634847055, Tokens per sec: 17993.972219222458, Loss: 2.282723903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 55.41862152219986, Tokens per sec: 17989.186354571208, Loss: 2.474703788757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 53.680784751998175, Tokens per sec: 17685.80677875881, Loss: 2.457491636276245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 54.91129861338375, Tokens per sec: 17902.48473040686, Loss: 2.361717700958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 56.46668425649344, Tokens per sec: 18251.423998657498, Loss: 2.3480582237243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 53.21077806657913, Tokens per sec: 17496.744155920085, Loss: 2.2280757427215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 52.14940769992199, Tokens per sec: 16723.461952230904, Loss: 2.5745627880096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 52.04805172935158, Tokens per sec: 16949.03021155367, Loss: 2.3547985553741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 47.243625161923795, Tokens per sec: 15607.464806745505, Loss: 2.426748752593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 53.96886678312184, Tokens per sec: 17942.412758747007, Loss: 2.3675312995910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 57.00247864623662, Tokens per sec: 17729.579614238784, Loss: 2.447209596633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 55.496715757551335, Tokens per sec: 18028.127106313812, Loss: 2.4042153358459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 53.51941554308538, Tokens per sec: 17544.63756716883, Loss: 2.402202844619751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 54.03676591152562, Tokens per sec: 17386.25659780541, Loss: 2.630662679672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 59.36844583988309, Tokens per sec: 18391.66371347833, Loss: 2.50496244430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 52.881846959374684, Tokens per sec: 17543.397372304866, Loss: 2.24295973777771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 45.575787365549765, Tokens per sec: 14892.698821760403, Loss: 2.460954427719116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 54.20670371613798, Tokens per sec: 17842.391366617794, Loss: 2.372318744659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 54.29981931705266, Tokens per sec: 17993.780498019405, Loss: 2.365842819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 55.40487147487553, Tokens per sec: 18367.824077738467, Loss: 2.183684825897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 54.41318371856707, Tokens per sec: 17887.565296992303, Loss: 2.458940267562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 53.93125625061047, Tokens per sec: 17459.580110165472, Loss: 2.50411319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 52.37893823889342, Tokens per sec: 16983.994239176125, Loss: 2.4960289001464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 54.34123698965633, Tokens per sec: 17786.961891794388, Loss: 2.4814207553863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 50.97770502289521, Tokens per sec: 16335.549909900872, Loss: 2.4644389152526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 55.53302164640677, Tokens per sec: 17562.91338008741, Loss: 2.489284038543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 46.54450321948463, Tokens per sec: 15335.97328118436, Loss: 2.360436201095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 53.7015162118194, Tokens per sec: 17297.059966966182, Loss: 2.4478542804718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 51.080644518718174, Tokens per sec: 16243.619873684522, Loss: 2.4493772983551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 54.73701011812291, Tokens per sec: 18095.329984509088, Loss: 2.3117146492004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 53.02599048610419, Tokens per sec: 17309.670165185886, Loss: 2.390876054763794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 48.770005440267894, Tokens per sec: 16133.727924466979, Loss: 2.3562192916870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 51.16088497362485, Tokens per sec: 16249.135448394767, Loss: 2.4109973907470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 56.989892387697566, Tokens per sec: 18497.679998903488, Loss: 2.36149263381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 56.29273841023965, Tokens per sec: 18211.909517100867, Loss: 2.432262420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 49.53759216899845, Tokens per sec: 16203.684859943553, Loss: 2.468301296234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 54.940846897538414, Tokens per sec: 17921.164495670902, Loss: 2.6919989585876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 53.67937566993398, Tokens per sec: 17827.81551705876, Loss: 2.383293390274048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 54.10702844772885, Tokens per sec: 17433.469982554685, Loss: 2.405768632888794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 55.37767613224889, Tokens per sec: 18226.274344349804, Loss: 2.377272129058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 53.90625395180698, Tokens per sec: 17880.18005733697, Loss: 2.4836251735687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 52.97903749828481, Tokens per sec: 17538.105579437655, Loss: 2.3171634674072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 52.069028735813475, Tokens per sec: 16507.80476931248, Loss: 2.53116512298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 53.91303331782742, Tokens per sec: 17876.30843031828, Loss: 2.233978271484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 55.10738355870683, Tokens per sec: 18237.99636721407, Loss: 2.4247350692749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 42.82788922111156, Tokens per sec: 14140.238503476821, Loss: 2.4334373474121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 51.96112363723876, Tokens per sec: 16491.133843826974, Loss: 2.318216323852539 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/model.pt +[rank0]:[2024-08-31 20:55:11,557] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006471825001426623, 'preprocessing_with_comm': 0.0007850590009184089, 'state_converting': 0.9832788260009693, : 0.9919720559992129}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000400, took 4.35s + eval ppl=13.28294563293457, eval loss=2.5864808559417725 +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 53.19086531050276, Tokens per sec: 17545.279777877255, Loss: 2.3276898860931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 54.82207359954914, Tokens per sec: 18160.63244986727, Loss: 2.3405702114105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 54.932967698133616, Tokens per sec: 17843.49758974676, Loss: 2.457974672317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 48.85405715989397, Tokens per sec: 15821.186223256303, Loss: 2.386705160140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 53.62050661434099, Tokens per sec: 17661.44281510632, Loss: 2.448812961578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 54.67847558500207, Tokens per sec: 17313.848891875834, Loss: 2.4315004348754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 53.66649326424295, Tokens per sec: 17572.041756980798, Loss: 2.4396309852600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 44.417288969899325, Tokens per sec: 14512.915119113635, Loss: 2.4858155250549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 52.901054916212075, Tokens per sec: 17361.019740435313, Loss: 2.2217302322387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 50.27947881480713, Tokens per sec: 15996.72563828664, Loss: 2.59730863571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 55.914060334009136, Tokens per sec: 18077.325679421534, Loss: 2.3200674057006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 51.668991981820334, Tokens per sec: 16671.49054922495, Loss: 2.316779136657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 53.10973235097099, Tokens per sec: 17548.39149495358, Loss: 2.245201349258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 41.71820494956612, Tokens per sec: 13790.309151827409, Loss: 2.422344923019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 54.313104494028266, Tokens per sec: 17476.619188462788, Loss: 2.4833908081054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 40.60048475523651, Tokens per sec: 12577.564591069038, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 53.83402735272707, Tokens per sec: 17576.404826312282, Loss: 2.423180103302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 52.08373592106873, Tokens per sec: 16316.245395223172, Loss: 2.371774911880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 51.90338790540361, Tokens per sec: 16834.021564862844, Loss: 2.4171295166015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 52.49414394729431, Tokens per sec: 17399.88257769614, Loss: 2.2325477600097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 54.19841419365143, Tokens per sec: 17931.044143489893, Loss: 2.3242764472961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 52.017798999833865, Tokens per sec: 17304.155233276946, Loss: 2.321336030960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 54.78899358063508, Tokens per sec: 17998.89276314975, Loss: 2.335113763809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 56.0180603571912, Tokens per sec: 17873.850910000558, Loss: 2.507380962371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 50.557535592288716, Tokens per sec: 16204.928688879234, Loss: 2.544006109237671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 49.770753807593486, Tokens per sec: 16083.10989866145, Loss: 2.3028998374938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 53.87224622763387, Tokens per sec: 17048.80623673732, Loss: 2.313508987426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 52.91533369520059, Tokens per sec: 17458.852472890056, Loss: 2.253291130065918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 54.34018000412646, Tokens per sec: 18059.66165145424, Loss: 2.2933454513549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 55.12752576809619, Tokens per sec: 17751.94594793812, Loss: 2.373188018798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 53.93376172943769, Tokens per sec: 17564.572508031662, Loss: 2.5704362392425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 56.58052462402858, Tokens per sec: 18301.958692272423, Loss: 2.5102672576904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 54.76371622541314, Tokens per sec: 18021.142628806236, Loss: 2.38147234916687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 49.38099724783505, Tokens per sec: 15719.902377314102, Loss: 2.365405321121216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 48.44518752252594, Tokens per sec: 16064.672422938318, Loss: 2.4309329986572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 53.959698150985545, Tokens per sec: 17750.53449506638, Loss: 2.4704432487487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 51.296354982204775, Tokens per sec: 16544.368129582224, Loss: 2.3032331466674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 52.07538479997074, Tokens per sec: 17012.254330675, Loss: 2.2926416397094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 54.17245946429196, Tokens per sec: 17919.397603194575, Loss: 2.3907976150512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 54.272782134783185, Tokens per sec: 17707.719854377556, Loss: 2.3636832237243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 50.40656411428325, Tokens per sec: 16440.74213647616, Loss: 2.339428186416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 54.27503092486138, Tokens per sec: 17705.47218443928, Loss: 2.373462200164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 53.392102441309014, Tokens per sec: 17711.15693631166, Loss: 2.2799720764160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 54.725199456953646, Tokens per sec: 17850.822419164557, Loss: 2.421154499053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 49.34324138731929, Tokens per sec: 16265.045644868214, Loss: 2.397277593612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 55.83495186256062, Tokens per sec: 18131.923443163694, Loss: 2.2442078590393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 52.57547055758661, Tokens per sec: 17166.941830799973, Loss: 2.3466296195983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 54.19420427652023, Tokens per sec: 17688.038960786238, Loss: 2.394613027572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 52.842846791713654, Tokens per sec: 17309.667564960426, Loss: 2.2336153984069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 53.58035230566498, Tokens per sec: 17697.856689118853, Loss: 2.2358341217041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 53.739821175325176, Tokens per sec: 17489.653606997057, Loss: 2.6158552169799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 57.163189034833096, Tokens per sec: 18360.166998138837, Loss: 2.5483644008636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 54.843751889252125, Tokens per sec: 18021.464727969727, Loss: 2.233564853668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 47.51898111376785, Tokens per sec: 15318.420359147864, Loss: 2.4007508754730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 51.90117758014999, Tokens per sec: 16823.44856735761, Loss: 2.419973373413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 55.557749498217184, Tokens per sec: 18171.33705241268, Loss: 2.356400489807129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 49.54319365617538, Tokens per sec: 16077.950221871337, Loss: 2.178039312362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 54.94020843111496, Tokens per sec: 18312.474587659755, Loss: 2.288186550140381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 54.38997803605621, Tokens per sec: 17394.47522564952, Loss: 2.3473801612854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 56.72958788261572, Tokens per sec: 18231.47667312236, Loss: 2.4772698879241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 56.736703201241255, Tokens per sec: 18421.67205282241, Loss: 2.3807132244110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 52.921133488144875, Tokens per sec: 17220.318871558815, Loss: 2.2281718254089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 54.443449980216926, Tokens per sec: 18189.13772456326, Loss: 2.3079941272735596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 49.788674584998944, Tokens per sec: 16462.284553778143, Loss: 2.4146711826324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 55.138146953414726, Tokens per sec: 18029.532084727616, Loss: 2.4516000747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 55.188654751216525, Tokens per sec: 18087.25437572402, Loss: 2.4544973373413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 49.967645278892704, Tokens per sec: 15864.951241933222, Loss: 2.4933316707611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 50.06099434748296, Tokens per sec: 16455.44862737901, Loss: 2.3410379886627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 50.722530181365364, Tokens per sec: 16375.649912140683, Loss: 2.237020969390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 57.24428063008515, Tokens per sec: 18383.165427198688, Loss: 2.4529240131378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 53.18151575282203, Tokens per sec: 17425.023679801852, Loss: 2.304063320159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 48.622958028645705, Tokens per sec: 16187.338990268283, Loss: 2.238271951675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 54.11953562145176, Tokens per sec: 17305.128695500298, Loss: 2.3544669151306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 55.8141054937582, Tokens per sec: 17824.91840555012, Loss: 2.430695056915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 54.14398801246628, Tokens per sec: 17597.54392927836, Loss: 2.3744425773620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 54.212522835604, Tokens per sec: 17864.032047346587, Loss: 2.4101414680480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 57.22691915338942, Tokens per sec: 18494.065567106623, Loss: 2.3582887649536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 53.675173938096314, Tokens per sec: 17471.548715302397, Loss: 2.551786184310913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 55.92965614986137, Tokens per sec: 18085.385560860876, Loss: 2.38887095451355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 53.2343740049112, Tokens per sec: 17319.337513749444, Loss: 2.335775852203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 54.17433767759981, Tokens per sec: 17554.406148309834, Loss: 2.4984257221221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 47.324772722335375, Tokens per sec: 15461.603383026495, Loss: 2.2882168292999268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 53.13298260502774, Tokens per sec: 17693.35633170966, Loss: 2.231428384780884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 53.10586362422862, Tokens per sec: 17216.820354797677, Loss: 2.457414150238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 53.96449240427264, Tokens per sec: 17920.97167366762, Loss: 2.2258408069610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 53.784552797868535, Tokens per sec: 17670.418938101866, Loss: 2.3827297687530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 54.07918529602532, Tokens per sec: 17287.946945179807, Loss: 2.4031083583831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 53.432988804450176, Tokens per sec: 17306.947079927333, Loss: 2.347884178161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 53.25580636543749, Tokens per sec: 16632.327034362323, Loss: 2.6014645099639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 51.98378181609237, Tokens per sec: 17062.8798673357, Loss: 2.4767186641693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 50.1219703766337, Tokens per sec: 16191.202833118106, Loss: 2.2983086109161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 52.679925223633745, Tokens per sec: 17385.622593399283, Loss: 2.3651576042175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 54.05651120351286, Tokens per sec: 17842.967643147327, Loss: 2.3882062435150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 54.78575087018449, Tokens per sec: 17795.678161364238, Loss: 2.405235767364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 54.649053165166414, Tokens per sec: 17567.21518948599, Loss: 2.3751797676086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 54.33404095958224, Tokens per sec: 18054.524631118784, Loss: 2.2422564029693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 55.14743259459212, Tokens per sec: 17588.816115553815, Loss: 2.390843391418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 49.399906558587084, Tokens per sec: 16108.340662116261, Loss: 2.3476433753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 55.05413133599153, Tokens per sec: 17946.03223541962, Loss: 2.3243281841278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 53.46560442976861, Tokens per sec: 17718.85497296856, Loss: 2.314136505126953 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/model.pt +[rank0]:[2024-08-31 20:56:47,189] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006483219000074314, 'preprocessing_with_comm': 0.0007288020005944418, 'state_converting': 0.969013134999841, : 0.9776872099992033}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000500, took 4.38s + eval ppl=13.333704948425293, eval loss=2.5902950763702393 +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 52.760479321981194, Tokens per sec: 17265.17376719258, Loss: 2.3375961780548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 54.3005629952715, Tokens per sec: 17570.33168261407, Loss: 2.312838554382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 53.24006074386199, Tokens per sec: 17698.596571194958, Loss: 2.3843462467193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 55.64426046641843, Tokens per sec: 18195.025310840636, Loss: 2.4002797603607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 53.81707675396859, Tokens per sec: 17785.143401822334, Loss: 2.3549914360046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 55.586534964925605, Tokens per sec: 18064.876526884855, Loss: 2.398667335510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 53.28714885716541, Tokens per sec: 17277.038057862646, Loss: 2.4419498443603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 56.3190058099456, Tokens per sec: 18634.213124706082, Loss: 2.303372621536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 53.67043148125685, Tokens per sec: 17458.273624128928, Loss: 2.2982912063598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 48.90854051937655, Tokens per sec: 16065.725978211938, Loss: 2.4136393070220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 55.28448840488155, Tokens per sec: 18086.54074737494, Loss: 2.3589932918548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 53.78528689488098, Tokens per sec: 17334.12743513557, Loss: 2.2708468437194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 39.25754019090223, Tokens per sec: 12676.316914746893, Loss: 2.29911208152771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 53.282069956362065, Tokens per sec: 17552.95289430428, Loss: 2.377647876739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 60.30240021463975, Tokens per sec: 18680.992068655134, Loss: 2.3226158618927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 57.62002358950573, Tokens per sec: 18890.489446698215, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 52.48784459319386, Tokens per sec: 16836.172796923, Loss: 2.3641390800476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 50.87295556150173, Tokens per sec: 16364.228883402058, Loss: 2.276500701904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 51.681337753425794, Tokens per sec: 17032.856265340317, Loss: 2.3794376850128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 49.063018976520645, Tokens per sec: 15444.859354593844, Loss: 2.270177125930786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 53.566239543343684, Tokens per sec: 17273.587887616886, Loss: 2.263397216796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 54.02473473003996, Tokens per sec: 17629.75743030259, Loss: 2.2238903045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 49.32988078288226, Tokens per sec: 15979.285814470628, Loss: 2.326805353164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 55.14527178142197, Tokens per sec: 18269.258382742537, Loss: 2.5155251026153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 56.90948461843059, Tokens per sec: 18765.49627182651, Loss: 2.393038749694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 53.08811793933798, Tokens per sec: 17637.495245693608, Loss: 2.244220733642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 55.324372367701294, Tokens per sec: 18358.41280839341, Loss: 2.31915020942688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 47.97294785794538, Tokens per sec: 15551.437777270376, Loss: 2.2351653575897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 55.27501197960352, Tokens per sec: 18354.60688080738, Loss: 2.2412428855895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 53.464433728640266, Tokens per sec: 17413.190143719363, Loss: 2.5000243186950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 53.555667656582656, Tokens per sec: 17434.113193368794, Loss: 2.4577062129974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 55.81867564783454, Tokens per sec: 18242.82121215537, Loss: 2.474127769470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 52.355367746159956, Tokens per sec: 17189.2150504328, Loss: 2.219639778137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 57.29858918031519, Tokens per sec: 18669.807514013974, Loss: 2.497809648513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 52.9209283973713, Tokens per sec: 17064.07345195433, Loss: 2.3451216220855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 54.92518727667429, Tokens per sec: 17846.954052549198, Loss: 2.327467441558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 45.672783522083336, Tokens per sec: 15140.146878334059, Loss: 2.2324447631835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 54.580371874097985, Tokens per sec: 17823.083911918675, Loss: 2.335975170135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 55.37281113054523, Tokens per sec: 18166.04479501284, Loss: 2.422548770904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 52.79445424259295, Tokens per sec: 17398.231536881944, Loss: 2.2783572673797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 51.89680278123561, Tokens per sec: 16826.25278256434, Loss: 2.3196284770965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 53.30990033510219, Tokens per sec: 17396.487298601554, Loss: 2.317903518676758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 54.01332063968978, Tokens per sec: 17549.187885501444, Loss: 2.325547218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 52.42600097119495, Tokens per sec: 16912.93028640953, Loss: 2.349977970123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 56.22339921878216, Tokens per sec: 18424.81815324913, Loss: 2.1880524158477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 55.581031790352576, Tokens per sec: 18212.769511463586, Loss: 2.356261730194092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 49.75499645826134, Tokens per sec: 16592.721204610843, Loss: 2.3065598011016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 53.84336749326901, Tokens per sec: 17909.90841860425, Loss: 2.290081024169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 53.14289081267616, Tokens per sec: 17581.833993703436, Loss: 2.210606336593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 49.809308536711946, Tokens per sec: 15971.70874439264, Loss: 2.4606282711029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 54.294299626622944, Tokens per sec: 17480.72864670461, Loss: 2.542262077331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 52.697773495193076, Tokens per sec: 17298.701901245702, Loss: 2.3675589561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 55.042888519734156, Tokens per sec: 17939.349431528644, Loss: 2.1988532543182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 56.77043233453619, Tokens per sec: 18427.991506184393, Loss: 2.391047716140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 46.98179778924762, Tokens per sec: 15478.741632748246, Loss: 2.3770556449890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 55.531882270285905, Tokens per sec: 18104.810840372927, Loss: 2.2761998176574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 57.35148832449091, Tokens per sec: 18537.414479830506, Loss: 2.2666125297546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 53.582128688418116, Tokens per sec: 17618.851500682624, Loss: 2.1476187705993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 51.26298707404042, Tokens per sec: 16439.201358210288, Loss: 2.49023175239563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 52.15317885940808, Tokens per sec: 16983.264017511417, Loss: 2.2757630348205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 53.38422161060062, Tokens per sec: 17636.080151919992, Loss: 2.347435474395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 53.36587008045221, Tokens per sec: 17741.941331860467, Loss: 2.2877371311187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 56.47552377207558, Tokens per sec: 17565.679927480756, Loss: 2.369159460067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 54.811703798575486, Tokens per sec: 17805.600737010933, Loss: 2.325979232788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 48.88051304930673, Tokens per sec: 16023.920979797624, Loss: 2.332822561264038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 53.852816857723596, Tokens per sec: 17327.071237682972, Loss: 2.5503060817718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 59.02841125347863, Tokens per sec: 18286.324897957216, Loss: 2.427544116973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 53.064020614579654, Tokens per sec: 17603.83294722874, Loss: 2.1689531803131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 49.57815912148642, Tokens per sec: 16200.544951895294, Loss: 2.384305238723755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 54.06570189432165, Tokens per sec: 17795.979954084254, Loss: 2.296504020690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 53.529468636393624, Tokens per sec: 17738.503017751154, Loss: 2.2910032272338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 55.175412034800324, Tokens per sec: 18291.75367966546, Loss: 2.1080780029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 54.2266328745398, Tokens per sec: 17826.23934295503, Loss: 2.381103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 44.586291629823656, Tokens per sec: 14434.26288660381, Loss: 2.428715944290161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 52.44970991005441, Tokens per sec: 17006.94212043735, Loss: 2.4227511882781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 54.657240652229305, Tokens per sec: 17890.39614201072, Loss: 2.407010793685913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 50.76049034278383, Tokens per sec: 16265.944555039781, Loss: 2.390249729156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 55.52495095326369, Tokens per sec: 17560.360936867382, Loss: 2.41243839263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 52.732869863173924, Tokens per sec: 17374.98151926267, Loss: 2.2933504581451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 54.363749752873446, Tokens per sec: 17510.362943861033, Loss: 2.37729549407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 51.32424597623494, Tokens per sec: 16321.085017553798, Loss: 2.3702447414398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 54.73903768677765, Tokens per sec: 18096.000271830126, Loss: 2.2450695037841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 52.82072850212401, Tokens per sec: 17242.66496249986, Loss: 2.313398838043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 47.535964695283944, Tokens per sec: 15725.491807871453, Loss: 2.2880382537841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 51.36387657022008, Tokens per sec: 16313.607318841536, Loss: 2.3381948471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 56.98779139197847, Tokens per sec: 18496.998061372768, Loss: 2.293339252471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 56.39693636494215, Tokens per sec: 18245.619792644397, Loss: 2.357393980026245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 55.26977876819495, Tokens per sec: 18078.675975678467, Loss: 2.3962996006011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 55.24290332545964, Tokens per sec: 18019.692334927666, Loss: 2.6196341514587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 53.74171244291855, Tokens per sec: 17848.518598546347, Loss: 2.3119471073150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 50.73711295938417, Tokens per sec: 16347.671664013518, Loss: 2.3254735469818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 49.290137953798535, Tokens per sec: 16222.702712756422, Loss: 2.308011531829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 54.10894878531736, Tokens per sec: 17947.411961878097, Loss: 2.4162449836730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 53.07673290437281, Tokens per sec: 17570.446528377182, Loss: 2.250920534133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 52.52693131559739, Tokens per sec: 16652.976795255, Loss: 2.460367441177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 54.14284726933296, Tokens per sec: 17952.509393719552, Loss: 2.162604331970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 55.33756738304006, Tokens per sec: 18314.176571768847, Loss: 2.3569068908691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 48.57614710091489, Tokens per sec: 16038.107832975365, Loss: 2.369352340698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 51.72795435268587, Tokens per sec: 16417.131866758962, Loss: 2.247087001800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 53.24144078184174, Tokens per sec: 17561.96235653734, Loss: 2.261857271194458 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/model.pt +[rank0]:[2024-08-31 20:58:22,490] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006543191000673687, 'preprocessing_with_comm': 0.0007405039996228879, 'state_converting': 0.9922303349994763, : 1.0009787750004762}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000600, took 4.40s + eval ppl=13.324822425842285, eval loss=2.5896286964416504 +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 45.9171799633682, Tokens per sec: 15210.753145535842, Loss: 2.277254819869995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 55.18940744423813, Tokens per sec: 17926.7951464467, Loss: 2.3921451568603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 53.143806272297404, Tokens per sec: 17210.403895316897, Loss: 2.321298837661743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 53.52532445706015, Tokens per sec: 17630.091857534717, Loss: 2.3844103813171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 54.628804181821074, Tokens per sec: 17298.120524182294, Loss: 2.362884998321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 53.561678528963704, Tokens per sec: 17537.72222549949, Loss: 2.374393939971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 49.54040399150975, Tokens per sec: 16186.842888646675, Loss: 2.428335189819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 52.77492343358995, Tokens per sec: 17319.62602601579, Loss: 2.1590731143951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 46.59136092278714, Tokens per sec: 14823.328231809839, Loss: 2.531939744949341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 55.59556455695578, Tokens per sec: 17974.354229039993, Loss: 2.2572779655456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 44.44425512792122, Tokens per sec: 14340.36064789466, Loss: 2.2553772926330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 53.158816414945925, Tokens per sec: 17564.60973467476, Loss: 2.1807069778442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 41.722707503769584, Tokens per sec: 13791.797509596247, Loss: 2.3627543449401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 54.28304467572642, Tokens per sec: 17466.946679365217, Loss: 2.419034481048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 45.56513332760969, Tokens per sec: 14115.555786677329, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 53.82953982705555, Tokens per sec: 17574.939682949436, Loss: 2.359063148498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 52.21871674509789, Tokens per sec: 16358.530769141951, Loss: 2.306312084197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 51.67475797043559, Tokens per sec: 16759.869155724555, Loss: 2.3531951904296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 52.269950349718236, Tokens per sec: 17325.57062632092, Loss: 2.173741340637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 48.28028997883211, Tokens per sec: 15973.087474067413, Loss: 2.263807773590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 53.47633553868359, Tokens per sec: 17789.349593033356, Loss: 2.2617757320404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 54.93205399668284, Tokens per sec: 18045.88995946265, Loss: 2.2754054069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 56.10762290905217, Tokens per sec: 17902.427902650328, Loss: 2.4433274269104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 51.98510287985864, Tokens per sec: 16662.498976327654, Loss: 2.4791419506073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 56.30691916565535, Tokens per sec: 18195.231129051466, Loss: 2.2453904151916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 54.26782433758445, Tokens per sec: 17173.993787290627, Loss: 2.2518203258514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 53.23382529152758, Tokens per sec: 17563.935393204938, Loss: 2.1968300342559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 54.27700886325693, Tokens per sec: 18038.6670682535, Loss: 2.2359485626220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 54.95395327045449, Tokens per sec: 17696.05282461628, Loss: 2.317207098007202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 47.89206869219968, Tokens per sec: 15596.978332861872, Loss: 2.51332426071167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 57.67536872456227, Tokens per sec: 18656.105134632136, Loss: 2.4493260383605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 55.06010319056975, Tokens per sec: 18118.674939258308, Loss: 2.328518867492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 49.629088685817024, Tokens per sec: 15798.879583184424, Loss: 2.309556722640991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 54.20565593713813, Tokens per sec: 17974.873266735212, Loss: 2.372596502304077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 54.31945766921733, Tokens per sec: 17868.880667434376, Loss: 2.4155025482177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 50.77246347871249, Tokens per sec: 16375.399907634612, Loss: 2.229973316192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 52.10514667067975, Tokens per sec: 17021.977091549514, Loss: 2.2356457710266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 54.28402979901614, Tokens per sec: 17956.303315219, Loss: 2.337676525115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 44.86144952361125, Tokens per sec: 14637.06022021458, Loss: 2.308788299560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 54.574817323361714, Tokens per sec: 17800.270947339504, Loss: 2.284273624420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 54.715601652686544, Tokens per sec: 17849.19412496817, Loss: 2.3185548782348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 53.35963312527652, Tokens per sec: 17700.38625814819, Loss: 2.226910352706909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 50.66848655015435, Tokens per sec: 16527.562523843964, Loss: 2.371289014816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 55.55209622164921, Tokens per sec: 18311.674614579344, Loss: 2.3468339443206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 55.600735849953466, Tokens per sec: 18055.86379471608, Loss: 2.1903939247131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 52.49420077826569, Tokens per sec: 17140.40562371694, Loss: 2.2958433628082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 49.5265559983868, Tokens per sec: 16164.600325584412, Loss: 2.344273567199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 51.90885971794227, Tokens per sec: 17003.72254612605, Loss: 2.1811208724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 47.57026301353273, Tokens per sec: 15712.694322619847, Loss: 2.1881213188171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 53.495325564765245, Tokens per sec: 17410.082379486386, Loss: 2.5669655799865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 53.268545396188934, Tokens per sec: 17109.251700880446, Loss: 2.498710870742798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 55.02249156112158, Tokens per sec: 18080.19795793891, Loss: 2.1823105812072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 48.95145104834156, Tokens per sec: 15780.197444752997, Loss: 2.3492302894592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 51.922958626100346, Tokens per sec: 16830.508759888373, Loss: 2.369663715362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 55.81858496088942, Tokens per sec: 18256.64880730281, Loss: 2.304236888885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 50.65697946413778, Tokens per sec: 16439.400331497327, Loss: 2.125964879989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 48.582427268557616, Tokens per sec: 16193.321615766481, Loss: 2.239011764526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 53.943818817096144, Tokens per sec: 17251.78891171589, Loss: 2.2993149757385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 57.10197428759287, Tokens per sec: 18351.152389254425, Loss: 2.423670768737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 57.507926770098656, Tokens per sec: 18672.078348276453, Loss: 2.332637071609497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 52.29175301901913, Tokens per sec: 17015.52105912525, Loss: 2.1797680854797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 53.79093196557387, Tokens per sec: 17971.136476655312, Loss: 2.2626287937164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 54.84118729897785, Tokens per sec: 18132.863308934702, Loss: 2.3653595447540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 54.979747942405474, Tokens per sec: 17977.73745235462, Loss: 2.4060604572296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 55.13325283223105, Tokens per sec: 18069.097227192124, Loss: 2.40743350982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 51.366743691157254, Tokens per sec: 16309.171256091457, Loss: 2.445010185241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 47.86777520568821, Tokens per sec: 15734.519980499217, Loss: 2.2936625480651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 49.79726084278038, Tokens per sec: 16076.928876164598, Loss: 2.190880537033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 56.61222738925753, Tokens per sec: 18180.190751702026, Loss: 2.4072718620300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 52.28126151865999, Tokens per sec: 17130.053686448795, Loss: 2.2590036392211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 54.28355548718145, Tokens per sec: 18071.839926940756, Loss: 2.1925506591796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 53.363594027061254, Tokens per sec: 17063.41068319673, Loss: 2.308838367462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 54.74710165323335, Tokens per sec: 17484.157656497337, Loss: 2.383638381958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 54.27094302128468, Tokens per sec: 17638.80606061987, Loss: 2.3300931453704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 53.78350620604199, Tokens per sec: 17722.663108612993, Loss: 2.363524913787842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 50.23926173023989, Tokens per sec: 16235.859176547283, Loss: 2.315293788909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 52.48306859791461, Tokens per sec: 17083.512217297237, Loss: 2.509384870529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 55.79564945021296, Tokens per sec: 18042.0532216742, Loss: 2.3441691398620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 53.59554785706075, Tokens per sec: 17436.842264456984, Loss: 2.2941770553588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 54.09538612307825, Tokens per sec: 17528.823045432604, Loss: 2.454387664794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 52.414571924443486, Tokens per sec: 17124.505327087998, Loss: 2.2424168586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 52.74022548271575, Tokens per sec: 17562.567669448013, Loss: 2.1862592697143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 53.11922588641903, Tokens per sec: 17221.152374879188, Loss: 2.4157347679138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 54.8704066170053, Tokens per sec: 18221.81510277897, Loss: 2.1854662895202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 56.17126864738603, Tokens per sec: 18454.552425381753, Loss: 2.338117837905884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 47.324196959990054, Tokens per sec: 15128.523142298123, Loss: 2.3603196144104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 53.9251513918078, Tokens per sec: 17466.358560450943, Loss: 2.306494951248169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 50.41573607286186, Tokens per sec: 15745.344353402463, Loss: 2.5595178604125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 52.54758110688166, Tokens per sec: 17247.93834580617, Loss: 2.4351072311401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 56.929602221607176, Tokens per sec: 18390.313266864036, Loss: 2.254547357559204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 50.74589781155469, Tokens per sec: 16747.34775665729, Loss: 2.3199446201324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 55.92327497710411, Tokens per sec: 18459.1488370129, Loss: 2.3505759239196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 55.640215959136064, Tokens per sec: 18073.22817175168, Loss: 2.367671489715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 48.059535314915586, Tokens per sec: 15448.98127021836, Loss: 2.3359973430633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 54.725747919731575, Tokens per sec: 18184.683972027324, Loss: 2.1982388496398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 55.726706564196526, Tokens per sec: 17773.570742424658, Loss: 2.353855848312378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 48.508760907759154, Tokens per sec: 15817.755543173269, Loss: 2.3110690116882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 55.747857756432936, Tokens per sec: 18172.166703472973, Loss: 2.2843899726867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 53.705282342255686, Tokens per sec: 17798.2858185912, Loss: 2.2728686332702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 54.021512859373, Tokens per sec: 17677.83042666793, Loss: 2.2986268997192383 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/model.pt +[rank0]:[2024-08-31 20:59:58,768] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006572121999852243, 'preprocessing_with_comm': 0.0007943250002426794, 'state_converting': 1.012361729000986, : 1.0211993210014043}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000700, took 4.60s + eval ppl=13.431185722351074, eval loss=2.59757924079895 +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 53.68826214858979, Tokens per sec: 17372.206131564446, Loss: 2.2742671966552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 52.75573802181356, Tokens per sec: 17537.593139793233, Loss: 2.3459300994873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 47.21235165758295, Tokens per sec: 15437.88930239192, Loss: 2.3631162643432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 53.1982071606567, Tokens per sec: 17580.62310588731, Loss: 2.3178961277008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 54.76933348333092, Tokens per sec: 17799.29703228404, Loss: 2.3603107929229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 52.43174229406944, Tokens per sec: 16999.693668783602, Loss: 2.405684471130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 55.70322105165063, Tokens per sec: 18430.469037608087, Loss: 2.267122745513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 52.711697298203866, Tokens per sec: 17146.41021557053, Loss: 2.2624731063842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 55.937439843011596, Tokens per sec: 18374.614553965323, Loss: 2.3764772415161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 54.472619383799845, Tokens per sec: 17820.934560992486, Loss: 2.3219223022460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 53.01261854324975, Tokens per sec: 17085.108931275056, Loss: 2.2333993911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 35.16807189380095, Tokens per sec: 11355.82164441715, Loss: 2.264509677886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 52.976401972951415, Tokens per sec: 17452.25531032343, Loss: 2.341722011566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 59.59545379369897, Tokens per sec: 18461.98817435653, Loss: 2.2791686058044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 56.64336390543209, Tokens per sec: 18570.29555739254, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 51.909004919748085, Tokens per sec: 16650.502289029613, Loss: 2.3271372318267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 55.50135598936358, Tokens per sec: 17853.039649939947, Loss: 2.240323066711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 56.051287524481054, Tokens per sec: 18473.08071719687, Loss: 2.3425745964050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 48.61734777649737, Tokens per sec: 15304.563686974072, Loss: 2.2337899208068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 52.751574326774694, Tokens per sec: 17010.881538667338, Loss: 2.2309138774871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 47.338527837734084, Tokens per sec: 15447.864150692909, Loss: 2.1905291080474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 53.31431687726529, Tokens per sec: 17269.952691243878, Loss: 2.2914955615997314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 54.42136009043255, Tokens per sec: 18029.431298719912, Loss: 2.479203224182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 52.7938458566766, Tokens per sec: 17408.39377199352, Loss: 2.3591904640197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 52.363984260273476, Tokens per sec: 17396.91590670968, Loss: 2.2121200561523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 51.99202179962111, Tokens per sec: 17252.63131042172, Loss: 2.284778118133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 53.10909077415916, Tokens per sec: 17216.426287319537, Loss: 2.2023868560791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 55.15797875057231, Tokens per sec: 18315.744855564364, Loss: 2.208805561065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 48.21310553824944, Tokens per sec: 15702.849831307813, Loss: 2.467635154724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 54.619308239949355, Tokens per sec: 17780.3628274203, Loss: 2.424570083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 50.913430242086406, Tokens per sec: 16639.67470428437, Loss: 2.440804958343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 52.76718744387503, Tokens per sec: 17324.42291260177, Loss: 2.1893744468688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 57.541873381418114, Tokens per sec: 18749.07768926207, Loss: 2.4646456241607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 52.82276761821119, Tokens per sec: 17032.42202790689, Loss: 2.314178943634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 55.51825323171727, Tokens per sec: 18039.660192930045, Loss: 2.290182590484619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 53.03217912253377, Tokens per sec: 17579.72515086722, Loss: 2.1980273723602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 52.72828905582226, Tokens per sec: 17218.29090028263, Loss: 2.3046176433563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 54.605428944244046, Tokens per sec: 17914.291292045884, Loss: 2.3904027938842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 52.32699354870372, Tokens per sec: 17244.181466597955, Loss: 2.2497663497924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 45.7834290981373, Tokens per sec: 14844.142798261384, Loss: 2.2891366481781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 53.59274650643421, Tokens per sec: 17488.787786805537, Loss: 2.287498950958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 54.630592198517, Tokens per sec: 17749.74238639016, Loss: 2.2970633506774902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 52.15364446807294, Tokens per sec: 16825.06650765433, Loss: 2.3217105865478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 56.02392211467032, Tokens per sec: 18359.448050763942, Loss: 2.158708333969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 55.56659601452274, Tokens per sec: 18208.039202409596, Loss: 2.3281989097595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 55.47627611735648, Tokens per sec: 18500.702414027935, Loss: 2.2778878211975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 53.83755243345874, Tokens per sec: 17907.974156437907, Loss: 2.26177978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 53.4309155087837, Tokens per sec: 17677.12430094133, Loss: 2.1817264556884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 50.89237610597166, Tokens per sec: 16319.002057126816, Loss: 2.433420181274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 48.189199064465846, Tokens per sec: 15515.11518411596, Loss: 2.5165741443634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 52.563434008744196, Tokens per sec: 17254.60328804988, Loss: 2.340622901916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 55.09755173181712, Tokens per sec: 17957.16503839409, Loss: 2.170323610305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 57.16331561599495, Tokens per sec: 18555.5235589787, Loss: 2.3639848232269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 53.68802315886722, Tokens per sec: 17688.191562548083, Loss: 2.3501267433166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 55.60678883029018, Tokens per sec: 18129.23229061262, Loss: 2.246598958969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 57.2408944318955, Tokens per sec: 18501.66780810719, Loss: 2.2396063804626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 53.45202282737977, Tokens per sec: 17576.070149864463, Loss: 2.120283603668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 51.730162904126175, Tokens per sec: 16589.017004523997, Loss: 2.4628663063049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 45.782531066780926, Tokens per sec: 14908.713706466557, Loss: 2.249467611312866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 53.82571232543046, Tokens per sec: 17781.931592629608, Loss: 2.322796106338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 54.02717518281868, Tokens per sec: 17961.797886451597, Loss: 2.2610700130462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 56.630676461862315, Tokens per sec: 17613.937337180585, Loss: 2.3436808586120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 55.60008135885955, Tokens per sec: 18061.705457273278, Loss: 2.2992701530456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 55.18707373741367, Tokens per sec: 18091.326246566958, Loss: 2.3092198371887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 53.96335577245178, Tokens per sec: 17362.63698450531, Loss: 2.5234241485595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 59.22458609182701, Tokens per sec: 18347.097613242375, Loss: 2.4029035568237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 52.66574367480719, Tokens per sec: 17471.705742518556, Loss: 2.1449644565582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 45.89353050233397, Tokens per sec: 14996.527040916642, Loss: 2.3604886531829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 54.49678079584277, Tokens per sec: 17937.871601123137, Loss: 2.272080659866333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 53.89471906950571, Tokens per sec: 17859.539076487385, Loss: 2.2678611278533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 54.424528035733594, Tokens per sec: 18042.820601571402, Loss: 2.0844366550445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 53.81020090415405, Tokens per sec: 17689.343216814763, Loss: 2.3561511039733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 54.14506771384022, Tokens per sec: 17528.798938545522, Loss: 2.4042694568634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 52.64577425514637, Tokens per sec: 17070.516446674403, Loss: 2.400073766708374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 54.219159960803566, Tokens per sec: 17747.003665218115, Loss: 2.384493589401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 51.06456330875765, Tokens per sec: 16363.383211991602, Loss: 2.367546558380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 48.76678986392831, Tokens per sec: 15423.02004847806, Loss: 2.3886585235595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 52.87907704515082, Tokens per sec: 17423.15540191733, Loss: 2.274139881134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 53.94544144511515, Tokens per sec: 17375.627383408028, Loss: 2.355452060699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 50.938816333849324, Tokens per sec: 16198.518580541284, Loss: 2.3462557792663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 54.48208183475992, Tokens per sec: 18011.054073203686, Loss: 2.2244839668273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 52.860637504973184, Tokens per sec: 17255.692756410128, Loss: 2.2893335819244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 53.65018531959137, Tokens per sec: 17748.152480804092, Loss: 2.267529010772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 51.25964036986912, Tokens per sec: 16280.501008444406, Loss: 2.316344976425171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 53.48204969807275, Tokens per sec: 17359.11052911484, Loss: 2.273064374923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 56.06871961179317, Tokens per sec: 18139.434625975395, Loss: 2.3354334831237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 49.31235751762631, Tokens per sec: 16130.01088527957, Loss: 2.3753747940063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 55.68795372685836, Tokens per sec: 18164.86340349902, Loss: 2.599564790725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 51.482146352759905, Tokens per sec: 17098.079032116875, Loss: 2.290192127227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 53.71993769251654, Tokens per sec: 17308.74801472314, Loss: 2.3009371757507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 54.94143753905392, Tokens per sec: 18082.69655570846, Loss: 2.2870090007781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 54.56285242063473, Tokens per sec: 18097.967382319854, Loss: 2.3968346118927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 53.101493386943226, Tokens per sec: 17578.643203477815, Loss: 2.2320895195007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 51.9925539821788, Tokens per sec: 16483.5594485636, Loss: 2.440796375274658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 53.85701929894225, Tokens per sec: 17857.735483919405, Loss: 2.1433920860290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 48.55349589886932, Tokens per sec: 16068.962535224777, Loss: 2.3383915424346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 49.06489877991723, Tokens per sec: 16199.47617112501, Loss: 2.3508524894714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 48.67219129759245, Tokens per sec: 15447.310700296499, Loss: 2.22526478767395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 53.231119357757514, Tokens per sec: 17558.557781856958, Loss: 2.241316795349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 54.03531374020312, Tokens per sec: 17900.00647033464, Loss: 2.2580597400665283 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/model.pt +[rank0]:[2024-08-31 21:01:34,845] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0065477149983053096, 'preprocessing_with_comm': 0.0007380860006378498, 'state_converting': 0.9881091939987527, : 0.9968825370015111}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000800, took 4.57s + eval ppl=13.429497718811035, eval loss=2.5974535942077637 +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 54.80969220199304, Tokens per sec: 17803.4548592985, Loss: 2.3740134239196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 52.99144735357764, Tokens per sec: 17161.06308380661, Loss: 2.3027868270874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 53.47323337168115, Tokens per sec: 17612.934173214144, Loss: 2.365710735321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 46.083161873683046, Tokens per sec: 14592.157016163304, Loss: 2.343021869659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 53.775711887204366, Tokens per sec: 17607.80325520034, Loss: 2.3553667068481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 52.719998593960185, Tokens per sec: 17225.744353565588, Loss: 2.410875082015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 52.96257668582641, Tokens per sec: 17381.209898428053, Loss: 2.1418538093566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 48.358816293126594, Tokens per sec: 15385.6550359793, Loss: 2.512478828430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 55.60850136042238, Tokens per sec: 17978.53676931912, Loss: 2.238618850708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 50.77405628213978, Tokens per sec: 16382.731053691576, Loss: 2.2384791374206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 52.718621936631116, Tokens per sec: 17419.161721712895, Loss: 2.1630077362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 45.39547450114972, Tokens per sec: 15005.862026459588, Loss: 2.3464794158935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 49.516603294035754, Tokens per sec: 15933.223249486573, Loss: 2.4016740322113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 39.888128208433315, Tokens per sec: 12356.884701818884, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 52.783422768860135, Tokens per sec: 17233.39033554357, Loss: 2.342515468597412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 51.32164531011289, Tokens per sec: 16077.505658108403, Loss: 2.289088487625122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 51.124570381159664, Tokens per sec: 16581.424739736314, Loss: 2.335568904876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 52.06319257114208, Tokens per sec: 17257.038009180567, Loss: 2.1577610969543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 53.650584668412385, Tokens per sec: 17749.79981104374, Loss: 2.2476534843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 53.34696372765194, Tokens per sec: 17746.312979721268, Loss: 2.245069980621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 54.29134278055265, Tokens per sec: 17835.40804114994, Loss: 2.2594213485717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 48.976401282514324, Tokens per sec: 15627.047581622375, Loss: 2.426541805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 51.24414946428485, Tokens per sec: 16425.005255154545, Loss: 2.4627349376678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 55.30386188959762, Tokens per sec: 17871.099401654817, Loss: 2.230576276779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 53.34680369890163, Tokens per sec: 16882.52083955813, Loss: 2.2358555793762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 52.41593114183576, Tokens per sec: 17294.079903297967, Loss: 2.1816036701202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 53.2548003650863, Tokens per sec: 17698.9416640903, Loss: 2.220623254776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 53.56437147609132, Tokens per sec: 17248.585238141546, Loss: 2.3025312423706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 53.09983347575265, Tokens per sec: 17292.98764525447, Loss: 2.497586488723755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 56.69557841097634, Tokens per sec: 18339.174848716706, Loss: 2.4345760345458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 54.56214945117783, Tokens per sec: 17954.812879145506, Loss: 2.31404972076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 47.57434476481287, Tokens per sec: 15144.774246136356, Loss: 2.2944436073303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 47.941206553976514, Tokens per sec: 15897.549751292612, Loss: 2.3565125465393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 53.60059786934134, Tokens per sec: 17632.40518458214, Loss: 2.4010097980499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 50.90936238344271, Tokens per sec: 16419.55325687705, Loss: 2.2116360664367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 50.63979443623514, Tokens per sec: 16543.268292907884, Loss: 2.2215588092803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 53.91668451376743, Tokens per sec: 17834.791272215498, Loss: 2.3243799209594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 53.923722705106954, Tokens per sec: 17593.831338806616, Loss: 2.2944507598876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 55.34205712415099, Tokens per sec: 18050.515968861284, Loss: 2.2700228691101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 55.16755379708133, Tokens per sec: 17996.628884284244, Loss: 2.3048899173736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 48.60447217318002, Tokens per sec: 16123.010615887617, Loss: 2.2130205631256104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 54.739380460741444, Tokens per sec: 17855.448123279908, Loss: 2.3573532104492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 56.403676737793106, Tokens per sec: 18592.381669404494, Loss: 2.333369731903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 56.02496281977797, Tokens per sec: 18193.627877656767, Loss: 2.176912307739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 52.506673745665466, Tokens per sec: 17144.478296838803, Loss: 2.282909393310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 52.64884230254924, Tokens per sec: 17183.659882450753, Loss: 2.3314011096954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 52.93679652579592, Tokens per sec: 17340.442565996804, Loss: 2.1681067943573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 54.17155187454303, Tokens per sec: 17893.13284529236, Loss: 2.1766154766082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 53.950588612228565, Tokens per sec: 17558.24798231231, Loss: 2.5548527240753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 57.60279824190557, Tokens per sec: 18501.364481905035, Loss: 2.4859302043914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 45.497521098364494, Tokens per sec: 14950.326034220836, Loss: 2.1695847511291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 53.66357671050346, Tokens per sec: 17299.218265197534, Loss: 2.336853265762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 52.44235861137994, Tokens per sec: 16998.86908128461, Loss: 2.3559906482696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 56.45214874626661, Tokens per sec: 18463.869243556455, Loss: 2.291985034942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 50.954172386848036, Tokens per sec: 16535.84653661661, Loss: 2.113736629486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 55.33526566413662, Tokens per sec: 18444.15365745162, Loss: 2.226471424102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 54.861569363621705, Tokens per sec: 17545.294989881288, Loss: 2.287677049636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 57.3740887441037, Tokens per sec: 18438.603198461115, Loss: 2.411755323410034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 57.69055389381038, Tokens per sec: 18731.37500795431, Loss: 2.320253849029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 42.445034933308015, Tokens per sec: 13811.439549565835, Loss: 2.1679582595825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 54.56538673505215, Tokens per sec: 18229.875856114257, Loss: 2.2510910034179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 55.2487532918643, Tokens per sec: 18267.622215558848, Loss: 2.353374719619751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 55.034669776885806, Tokens per sec: 17995.69625277916, Loss: 2.3947434425354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 55.201845738625906, Tokens per sec: 18091.577524128632, Loss: 2.3945505619049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 51.21025888331851, Tokens per sec: 16259.486628516786, Loss: 2.4330880641937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 53.496582583086564, Tokens per sec: 17584.753916910184, Loss: 2.281327486038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 50.84905556058078, Tokens per sec: 16416.498334086984, Loss: 2.1798884868621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 56.753395059748414, Tokens per sec: 18225.524689189617, Loss: 2.395674228668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 52.55131285349975, Tokens per sec: 17218.536514321095, Loss: 2.247560977935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 48.14086300121501, Tokens per sec: 16026.842057318505, Loss: 2.1818552017211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 54.18212800626082, Tokens per sec: 17325.143081470927, Loss: 2.2979495525360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 55.45503969493279, Tokens per sec: 17710.246343538798, Loss: 2.372807025909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 54.139838315632204, Tokens per sec: 17596.19522049256, Loss: 2.319446325302124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 53.75487917342599, Tokens per sec: 17713.229970265536, Loss: 2.3526535034179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 56.92776259782758, Tokens per sec: 18397.38692294336, Loss: 2.306067943572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 46.69980691840846, Tokens per sec: 15201.030415126164, Loss: 2.4989871978759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 51.310292259381555, Tokens per sec: 16591.67037009704, Loss: 2.3345179557800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 47.35089454958598, Tokens per sec: 15405.198983021392, Loss: 2.283690929412842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 54.31315578028643, Tokens per sec: 17599.388135349545, Loss: 2.4439239501953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 52.703463955676185, Tokens per sec: 17218.89001718771, Loss: 2.232016086578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 52.910156627730274, Tokens per sec: 17619.154974605484, Loss: 2.1770215034484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 52.91927825631317, Tokens per sec: 17156.329732086695, Loss: 2.407140016555786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 54.546407585521855, Tokens per sec: 18114.218844446506, Loss: 2.1768064498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 56.12468500985973, Tokens per sec: 18439.247800765017, Loss: 2.32820200920105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 55.362088735568555, Tokens per sec: 17698.06345261621, Loss: 2.3499755859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 54.27499615297073, Tokens per sec: 17579.67329172669, Loss: 2.2965259552001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 54.33005401664667, Tokens per sec: 16967.825442333145, Loss: 2.5490145683288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 47.41431230171058, Tokens per sec: 15563.021514259648, Loss: 2.42513370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 56.73797746097569, Tokens per sec: 18328.411563002082, Loss: 2.2460131645202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 53.50142138775482, Tokens per sec: 17656.735777609476, Loss: 2.3113274574279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 56.01550282845158, Tokens per sec: 18489.59139667411, Loss: 2.341146469116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 55.452413417006355, Tokens per sec: 18012.225565334084, Loss: 2.3597753047943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 52.97957880523788, Tokens per sec: 17030.55419289826, Loss: 2.327345848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 52.31191489900806, Tokens per sec: 17382.597343490357, Loss: 2.1885645389556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 55.939195425938756, Tokens per sec: 17841.34230204127, Loss: 2.3467202186584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 49.817893668375895, Tokens per sec: 16244.638060753367, Loss: 2.303154230117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 46.89693192027321, Tokens per sec: 15287.024453209366, Loss: 2.275942802429199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 53.99852494184786, Tokens per sec: 17895.468355840832, Loss: 2.2644965648651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 53.90409147132802, Tokens per sec: 17639.405819942705, Loss: 2.2900006771087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 54.09496635966619, Tokens per sec: 17503.805648975533, Loss: 2.2669644355773926 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/model.pt +[rank0]:[2024-08-31 21:03:11,064] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006519807000586297, 'preprocessing_with_comm': 0.0007181260007200763, 'state_converting': 1.0151290829999198, : 1.0238383209998574}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0000900, took 4.72s + eval ppl=13.448776245117188, eval loss=2.5988881587982178 +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 52.30829358232539, Tokens per sec: 17388.849158064422, Loss: 2.3385303020477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 55.14049035925036, Tokens per sec: 18030.29835115178, Loss: 2.356720209121704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 50.289393528511845, Tokens per sec: 16619.335895632863, Loss: 2.3106191158294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 54.67571856663001, Tokens per sec: 17768.873442967968, Loss: 2.352811336517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 52.56736156423475, Tokens per sec: 17043.664857752752, Loss: 2.398486614227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 47.59436229364379, Tokens per sec: 15747.499050446928, Loss: 2.2599453926086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 52.820316567068545, Tokens per sec: 17181.74261116259, Loss: 2.2560932636260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 56.23506049367187, Tokens per sec: 18472.37849801658, Loss: 2.369431972503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 52.90216642971998, Tokens per sec: 17307.154617190154, Loss: 2.314732551574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 53.24784810004947, Tokens per sec: 17160.919610170306, Loss: 2.227020740509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 43.698648790324015, Tokens per sec: 14110.35735093614, Loss: 2.2579522132873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 52.6169139143871, Tokens per sec: 17333.827535966837, Loss: 2.3361380100250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 59.11464921106415, Tokens per sec: 18313.04042828334, Loss: 2.2714240550994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 57.263449092060014, Tokens per sec: 18773.587953756327, Loss: nan +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 51.76390017085957, Tokens per sec: 16603.95801492433, Loss: 2.320413112640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 47.4512377780263, Tokens per sec: 15263.569950474392, Loss: 2.2324588298797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 55.47071621308801, Tokens per sec: 18281.739158936172, Loss: 2.3362040519714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 48.38876036843598, Tokens per sec: 15232.605204977095, Loss: 2.227795124053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 53.061412270790086, Tokens per sec: 17110.795458376742, Loss: 2.2243576049804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 53.263423681543756, Tokens per sec: 17381.320687742624, Loss: 2.1841700077056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 53.20603663614051, Tokens per sec: 17234.877785455883, Loss: 2.2853736877441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 54.08143447946038, Tokens per sec: 17916.81622552964, Loss: 2.473278760910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 53.651856627485394, Tokens per sec: 17691.316698264145, Loss: 2.352590322494507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 46.24329365052454, Tokens per sec: 15363.435427082142, Loss: 2.2067415714263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 54.53789224195496, Tokens per sec: 18097.43331244746, Loss: 2.2790637016296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 52.79627980194451, Tokens per sec: 17115.02204622832, Loss: 2.1966516971588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 50.92736084423669, Tokens per sec: 16910.926914279284, Loss: 2.201629877090454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 52.89484050997301, Tokens per sec: 17227.675506613745, Loss: 2.4624080657958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 54.85331895766741, Tokens per sec: 17856.54093367264, Loss: 2.419358253479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 56.2935689304134, Tokens per sec: 18398.027209940017, Loss: 2.435178756713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 52.864736260556214, Tokens per sec: 17356.4499551006, Loss: 2.1839895248413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 57.32930848785287, Tokens per sec: 18679.816897611316, Loss: 2.4583983421325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 52.83934998765649, Tokens per sec: 17037.768925226937, Loss: 2.3083648681640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 49.84252737982476, Tokens per sec: 16195.43492004492, Loss: 2.283457040786743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 49.427199231103536, Tokens per sec: 16384.704378303522, Loss: 2.1920197010040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 54.06164040429322, Tokens per sec: 17653.69344797296, Loss: 2.2996747493743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 54.48195589992193, Tokens per sec: 17873.783743154352, Loss: 2.385051727294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 52.46877564635462, Tokens per sec: 17290.905271173724, Loss: 2.2449302673339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 51.84981693236447, Tokens per sec: 16811.018784939442, Loss: 2.284125804901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 53.45623798366797, Tokens per sec: 17444.24129980169, Loss: 2.283583641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 51.643154438603716, Tokens per sec: 16779.109477247457, Loss: 2.2928924560546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 52.22878283024232, Tokens per sec: 16849.30657665948, Loss: 2.3167402744293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 50.179094821285084, Tokens per sec: 16444.05550043598, Loss: 2.154710054397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 55.22388877804856, Tokens per sec: 18095.741036888703, Loss: 2.3223862648010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 54.97503544595118, Tokens per sec: 18333.544393546174, Loss: 2.2722344398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 53.99173119160226, Tokens per sec: 17959.2586055168, Loss: 2.257368564605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 53.63982495014276, Tokens per sec: 17746.240057753268, Loss: 2.1770052909851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 46.65059612554999, Tokens per sec: 14958.845162855609, Loss: 2.4284212589263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 55.11141499302568, Tokens per sec: 17743.80915591768, Loss: 2.512453556060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 52.98708618922271, Tokens per sec: 17393.672404140314, Loss: 2.336580514907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 55.05747982884993, Tokens per sec: 17944.10497034448, Loss: 2.1658525466918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 53.62106860425528, Tokens per sec: 17405.69088797692, Loss: 2.359865665435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 47.68733851765517, Tokens per sec: 15711.190861179022, Loss: 2.3459982872009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 55.26286992489465, Tokens per sec: 18017.105950354464, Loss: 2.24237060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 53.02545766972588, Tokens per sec: 17139.134755229414, Loss: 2.23469614982605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 48.48506817587771, Tokens per sec: 15942.838351174174, Loss: 2.114994764328003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 52.04952696846043, Tokens per sec: 16691.431835571286, Loss: 2.4594733715057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 48.39598861847553, Tokens per sec: 15759.765177722786, Loss: 2.2457432746887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 50.9398122454273, Tokens per sec: 16828.541928308612, Loss: 2.3189425468444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 53.76736324541454, Tokens per sec: 17875.421178945617, Loss: 2.257432699203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 49.84973250933691, Tokens per sec: 15504.848600669518, Loss: 2.3393661975860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 54.98477104627199, Tokens per sec: 17861.821691653422, Loss: 2.2962636947631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 55.243901135741005, Tokens per sec: 18109.955301040383, Loss: 2.306002616882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 53.88088163770548, Tokens per sec: 17336.101042814633, Loss: 2.5196850299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 59.21489987181965, Tokens per sec: 18344.09693336083, Loss: 2.3988420963287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 53.01697719345394, Tokens per sec: 17588.226430474606, Loss: 2.1411538124084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 51.94038799604821, Tokens per sec: 16972.4452351476, Loss: 2.3569395542144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 54.63205548712099, Tokens per sec: 17982.39790171547, Loss: 2.268615245819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 54.65851574464942, Tokens per sec: 18112.644701708807, Loss: 2.264294385910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 54.954867738706334, Tokens per sec: 18218.63883029991, Loss: 2.0803606510162354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 47.77995332819711, Tokens per sec: 15706.984532752822, Loss: 2.3524231910705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 54.4018879804238, Tokens per sec: 17611.941337407767, Loss: 2.4008853435516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 52.6402941511171, Tokens per sec: 17068.73951002023, Loss: 2.3973677158355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 54.24789702382342, Tokens per sec: 17756.409874445806, Loss: 2.381171703338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 50.939903381215586, Tokens per sec: 16323.436563408395, Loss: 2.363837718963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 55.30169553830939, Tokens per sec: 17489.754018708954, Loss: 2.3852179050445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 52.82325612903902, Tokens per sec: 17404.76293838657, Loss: 2.2713608741760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 53.89216706635473, Tokens per sec: 17358.467902836062, Loss: 2.3524999618530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 51.45538782810258, Tokens per sec: 16362.7880620502, Loss: 2.3421974182128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 48.46995263652318, Tokens per sec: 16023.523853397692, Loss: 2.2211875915527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 53.02694349890144, Tokens per sec: 17309.981264272057, Loss: 2.2862298488616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 54.13874170601041, Tokens per sec: 17909.7730453927, Loss: 2.2650442123413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 51.24801701091248, Tokens per sec: 16276.809330043041, Loss: 2.315046787261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 57.18999666788152, Tokens per sec: 18562.629497598326, Loss: 2.270660400390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 56.557768940713835, Tokens per sec: 18297.652584085634, Loss: 2.3321266174316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 55.32809904194923, Tokens per sec: 18097.752464774585, Loss: 2.372748374938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 55.478967800630656, Tokens per sec: 18096.69424753039, Loss: 2.5965118408203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 53.74052894304147, Tokens per sec: 17848.125538507746, Loss: 2.2878711223602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 54.362631653760666, Tokens per sec: 17515.826211452106, Loss: 2.2981152534484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 49.72970576792874, Tokens per sec: 16367.376237050768, Loss: 2.285003662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 54.71945784560412, Tokens per sec: 18149.911878387262, Loss: 2.394017457962036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 52.984345092136536, Tokens per sec: 17539.862597793213, Loss: 2.2300658226013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 50.49174541529875, Tokens per sec: 16007.747715184232, Loss: 2.4387359619140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 53.57508803273876, Tokens per sec: 17764.253630633713, Loss: 2.1403963565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 55.241397352218435, Tokens per sec: 18282.348737466185, Loss: 2.3356752395629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 49.269802066052335, Tokens per sec: 16267.127913687851, Loss: 2.3485374450683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 51.65327135287667, Tokens per sec: 16393.429389608733, Loss: 2.2228846549987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 45.525823197038996, Tokens per sec: 15016.92631709273, Loss: 2.2393932342529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 54.86534593857996, Tokens per sec: 18174.967059866245, Loss: 2.256662368774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 54.75339977192613, Tokens per sec: 17785.169776909734, Loss: 2.3717682361602783 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/model.pt +[rank0]:[2024-08-31 21:04:47,156] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006692427999951178, 'preprocessing_with_comm': 0.0007078479993651854, 'state_converting': 0.9778827350000938, : 0.9867110420000245}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000, took 4.52s + eval ppl=13.448174476623535, eval loss=2.5988433361053467 +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/model.pt +[rank0]:[2024-08-31 21:04:52,387] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006588518999706139, 'preprocessing_with_comm': 0.0007461460008926224, 'state_converting': 0.9938468310010649, : 1.002643634999913}) +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/optimizer.pt +Saving sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/sampler.pt +Saved sampler indices to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/sampler.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara/iter_0001000, took 4.57s \ No newline at end of file diff --git a/wandb/run-20240831_204827-vaha0m5b/files/requirements.txt b/wandb/run-20240831_204827-vaha0m5b/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240831_204827-vaha0m5b/files/wandb-metadata.json b/wandb/run-20240831_204827-vaha0m5b/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6f65466aaff85f5f82007b1a4be8c001024bee27 --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/files/wandb-metadata.json @@ -0,0 +1,223 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-31T11:48:28.558645", + "startedAt": "2024-08-31T11:48:27.844471", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "4", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "16", + "--train-iters", + "1000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--instruction-train-data-path", + "/work/datasets/bin/ichikara/train/data", + "--instruction-valid-data-path", + "/work/datasets/bin/ichikara/valid/data", + "--efficient-instruction-tuning", + "--remove-padding-masking", + "--source-key", + "source", + "--target-key", + "target", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "25", + "--lr-decay-iters", + "1000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.99", + "--adam-eps", + "1e-6", + "--save-interval", + "100", + "--eval-interval", + "100", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--hf-transformer-model-dir", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "SHARD_GRAD_OP", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-sampler-state", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara", + "--num-workers", + "4", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara_train_2024-08-31-20:48:03" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.025999999999, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.026, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48781967163086 + } +} diff --git a/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json b/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1071c5c8340ef90a9bab75dbdfe53d48d426cbf0 --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.3717682361602783, "training/perplexity": 10.716324533379485, "utils/batch_size": 9, "utils/global_batch_size": 36, "utils/seq_len": 445, "utils/gradient_accumulation_steps": 4, "utils/iteration": 1000, "optimizer/lr": 1e-06, "optimizer/variance_l2": 0.008815568820036375, "optimizer/variance_sqrt_l2": 1.0142780468687387, "optimizer/momentum_l2": 0.22287244506672316, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 1.02825927734375, "optimizer/variance_sqrt_l1": 10542.0, "optimizer/momentum_l1": 1905.25, "optimizer/weight_l1": 6887424.0, "optimizer/variance_abs_max": 0.003814697265625, "optimizer/variance_sqrt_abs_max": 0.061767578125, "optimizer/momentum_abs_max": 0.01513671875, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 0.9007504679993872, "stats/tokens_per_sec": 17785.169776909734, "stats/tokens_per_sec_per_gpu": 17785.169776909734, "stats/tflops": 54.75339977192613, "_timestamp": 1725105889.7743924, "_runtime": 981.9163782596588, "_step": 1000, "evaluation/val_loss": 2.5988433361053467, "evaluation/val_ppl": 13.448174476623535, "_wandb": {"runtime": 985}} \ No newline at end of file diff --git a/wandb/run-20240831_204827-vaha0m5b/logs/debug-internal.log b/wandb/run-20240831_204827-vaha0m5b/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a221beb62404b8143abb3127c2f9b9ff0436e5d0 --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/logs/debug-internal.log @@ -0,0 +1,5905 @@ +2024-08-31 20:48:27,860 INFO StreamThr :15237 [internal.py:wandb_internal():86] W&B internal server running at pid: 15237, started at: 2024-08-31 20:48:27.859683 +2024-08-31 20:48:27,862 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status +2024-08-31 20:48:27,863 INFO WriterThread:15237 [datastore.py:open_for_write():87] open: /project/wandb/run-20240831_204827-vaha0m5b/run-vaha0m5b.wandb +2024-08-31 20:48:27,864 DEBUG SenderThread:15237 [sender.py:send():382] send: header +2024-08-31 20:48:28,043 DEBUG SenderThread:15237 [sender.py:send():382] send: run +2024-08-31 20:48:28,461 INFO SenderThread:15237 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240831_204827-vaha0m5b/files +2024-08-31 20:48:28,461 INFO SenderThread:15237 [sender.py:_start_run_threads():1136] run started: vaha0m5b with start time 1725104907.858014 +2024-08-31 20:48:28,466 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: check_version +2024-08-31 20:48:28,466 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: check_version +2024-08-31 20:48:28,538 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: run_start +2024-08-31 20:48:28,544 DEBUG HandlerThread:15237 [system_info.py:__init__():27] System info init +2024-08-31 20:48:28,544 DEBUG HandlerThread:15237 [system_info.py:__init__():42] System info init done +2024-08-31 20:48:28,544 INFO HandlerThread:15237 [system_monitor.py:start():194] Starting system monitor +2024-08-31 20:48:28,544 INFO SystemMonitor:15237 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-31 20:48:28,545 INFO HandlerThread:15237 [system_monitor.py:probe():214] Collecting system info +2024-08-31 20:48:28,545 INFO SystemMonitor:15237 [interfaces.py:start():190] Started cpu monitoring +2024-08-31 20:48:28,546 INFO SystemMonitor:15237 [interfaces.py:start():190] Started disk monitoring +2024-08-31 20:48:28,547 INFO SystemMonitor:15237 [interfaces.py:start():190] Started gpu monitoring +2024-08-31 20:48:28,547 INFO SystemMonitor:15237 [interfaces.py:start():190] Started memory monitoring +2024-08-31 20:48:28,548 INFO SystemMonitor:15237 [interfaces.py:start():190] Started network monitoring +2024-08-31 20:48:28,558 DEBUG HandlerThread:15237 [system_info.py:probe():151] Probing system +2024-08-31 20:48:28,560 DEBUG HandlerThread:15237 [system_info.py:_probe_git():136] Probing git +2024-08-31 20:48:28,573 DEBUG HandlerThread:15237 [system_info.py:_probe_git():144] Probing git done +2024-08-31 20:48:28,573 DEBUG HandlerThread:15237 [system_info.py:probe():199] Probing system done +2024-08-31 20:48:28,573 DEBUG HandlerThread:15237 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-31T11:48:28.558645', 'startedAt': '2024-08-31T11:48:27.844471', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '4', '--valid_micro_batch_size', '1', '--global-batch-size', '16', '--train-iters', '1000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--instruction-train-data-path', '/work/datasets/bin/ichikara/train/data', '--instruction-valid-data-path', '/work/datasets/bin/ichikara/valid/data', '--efficient-instruction-tuning', '--remove-padding-masking', '--source-key', 'source', '--target-key', 'target', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '25', '--lr-decay-iters', '1000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.99', '--adam-eps', '1e-6', '--save-interval', '100', '--eval-interval', '100', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--hf-transformer-model-dir', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara', '--fsdp-activation-checkpointing', '--sharding-strategy', 'SHARD_GRAD_OP', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-sampler-state', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara', '--num-workers', '4', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara_train_2024-08-31-20:48:03'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.025999999999, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}, {'current': 2400.026, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48781967163086}} +2024-08-31 20:48:28,573 INFO HandlerThread:15237 [system_monitor.py:probe():224] Finished collecting system info +2024-08-31 20:48:28,573 INFO HandlerThread:15237 [system_monitor.py:probe():227] Publishing system info +2024-08-31 20:48:28,575 INFO HandlerThread:15237 [system_monitor.py:probe():229] Finished publishing system info +2024-08-31 20:48:28,580 DEBUG SenderThread:15237 [sender.py:send():382] send: files +2024-08-31 20:48:28,580 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-31 20:48:28,592 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: python_packages +2024-08-31 20:48:28,592 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:48:28,592 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: python_packages +2024-08-31 20:48:28,593 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:48:28,594 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:48:28,844 DEBUG SenderThread:15237 [sender.py:send():382] send: telemetry +2024-08-31 20:48:29,293 INFO wandb-upload_0:15237 [upload_job.py:push():131] Uploaded file /tmp/tmp7h4p53c0wandb/f96g6zy8-wandb-metadata.json +2024-08-31 20:48:29,462 INFO Thread-12 :15237 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:29,463 INFO Thread-12 :15237 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-metadata.json +2024-08-31 20:48:29,463 INFO Thread-12 :15237 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_204827-vaha0m5b/files/requirements.txt +2024-08-31 20:48:31,463 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:33,464 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:33,487 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:48:34,071 DEBUG SenderThread:15237 [sender.py:send():382] send: config +2024-08-31 20:48:34,072 DEBUG SenderThread:15237 [sender.py:send():382] send: config +2024-08-31 20:48:35,466 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:39,072 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:48:43,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:48:43,660 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:48:43,702 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:48:44,926 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:48:49,926 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:48:51,821 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:52,718 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:52,722 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:52,722 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:52,724 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:53,478 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:53,478 INFO Thread-12 :15237 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:53,622 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:53,625 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:53,626 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:53,627 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:54,479 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:54,515 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:54,517 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:54,518 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:54,519 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:55,413 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:55,416 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:55,416 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:55,416 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:48:55,417 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:55,481 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:55,481 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:56,320 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:56,323 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:56,323 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:56,324 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:56,482 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:57,211 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:57,214 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:57,214 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:57,215 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:57,483 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:57,484 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:58,197 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:58,199 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:58,200 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:58,201 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:58,485 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:58,643 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:48:58,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:48:58,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:48:59,079 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:59,081 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:59,081 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:59,082 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:48:59,486 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:48:59,486 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:48:59,956 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:48:59,959 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:48:59,959 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:48:59,960 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:00,964 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:01,358 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:01,487 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:02,227 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:03,115 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:03,999 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:04,593 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:04,593 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:04,595 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:04,595 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:04,595 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:04,596 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:04,596 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:04,597 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:04,597 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:04,598 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:04,598 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:04,599 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:04,891 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:04,893 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:04,894 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:04,895 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:05,491 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:05,491 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:05,491 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/config.yaml +2024-08-31 20:49:05,897 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:05,899 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:05,900 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:05,901 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:06,492 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:06,789 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:06,791 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:06,792 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:06,795 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:07,493 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:07,493 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:07,686 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:07,689 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:07,689 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:07,690 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:08,494 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:08,554 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:08,556 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:08,556 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:08,558 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:09,496 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:09,496 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:09,500 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:09,503 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:09,503 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:09,504 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:10,385 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:10,387 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:10,388 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:10,388 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:10,389 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:10,497 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:11,286 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:11,289 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:11,290 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:11,291 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:11,498 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:11,498 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:12,154 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:12,156 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:12,156 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:12,158 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:12,499 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:13,037 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:13,039 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:13,040 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:13,041 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:13,500 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:13,501 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:13,643 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:49:13,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:49:13,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:49:13,944 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:13,947 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:13,947 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:13,948 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:14,832 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:14,834 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:14,834 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:14,836 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:15,503 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:15,503 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:15,836 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:15,882 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:15,884 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:15,885 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:15,886 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:16,504 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:16,778 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:16,806 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:16,807 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:16,809 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:17,505 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:17,505 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:17,668 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:17,670 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:17,671 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:17,672 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:18,506 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:18,559 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:18,561 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:18,562 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:18,564 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:19,454 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:19,459 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:19,460 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:19,461 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:19,507 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:19,508 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:20,321 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:20,324 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:20,324 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:20,325 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:20,509 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:21,182 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:21,184 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:21,185 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:21,185 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:21,187 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:21,510 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:21,510 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:22,048 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:22,050 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:22,050 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:22,052 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:22,511 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:23,065 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:23,068 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:23,068 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:23,069 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:23,512 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:23,513 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:23,933 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:23,936 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:23,936 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:23,938 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:24,826 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:24,828 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:24,829 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:24,830 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:25,515 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:25,515 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:25,694 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:25,697 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:25,697 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:25,698 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:26,516 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:26,631 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:26,634 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:26,634 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:26,634 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:26,635 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:27,516 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:27,519 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:27,519 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:27,520 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:27,520 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:27,521 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:28,445 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:28,448 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:28,448 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:28,449 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:28,521 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:28,548 DEBUG SystemMonitor:15237 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-31 20:49:28,549 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:49:28,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:49:28,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:49:28,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:49:29,330 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:29,332 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:29,332 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:29,333 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:29,523 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:29,523 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:30,220 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:30,222 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:30,223 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:30,224 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:30,524 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:31,100 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:31,103 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:31,103 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:31,104 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:31,525 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:31,525 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:32,105 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:32,106 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:32,109 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:32,109 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:32,110 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:32,526 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:32,995 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:32,997 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:32,998 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:32,999 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:33,527 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:33,885 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:33,888 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:33,888 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:33,889 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:34,529 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:34,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:34,778 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:34,778 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:34,780 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:35,530 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:35,530 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:35,668 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:35,670 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:35,670 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:35,671 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:36,531 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:36,579 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:36,582 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:36,582 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:36,583 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:37,455 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:37,457 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:37,458 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:37,458 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:37,459 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:37,533 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:37,533 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:38,345 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:38,348 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:38,348 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:38,349 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:38,534 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:39,280 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:39,282 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:39,282 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:39,284 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:39,536 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:39,536 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:40,281 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:40,283 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:40,284 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:40,285 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:40,538 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:41,169 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:41,171 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:41,172 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:41,174 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:41,539 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:41,539 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:42,043 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:42,045 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:42,045 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:42,047 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:42,541 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:42,956 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:42,959 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:42,959 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:42,959 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:42,960 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:43,542 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:43,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:49:43,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:49:43,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:49:43,901 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:43,903 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:43,904 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:43,907 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:44,543 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:44,830 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:44,833 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:44,833 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:44,834 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:45,544 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:45,545 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:45,740 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:45,742 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:45,743 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:45,744 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:46,545 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:46,656 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:46,658 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:46,659 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:46,660 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:47,546 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:47,549 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:47,549 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:47,550 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:47,550 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:47,550 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:48,513 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:48,516 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:48,516 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:48,516 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:48,517 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:48,551 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:49,520 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:49,522 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:49,523 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:49,524 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:49,552 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:49,552 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:50,400 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:50,403 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:50,403 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:50,404 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:50,553 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:51,270 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:51,272 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:51,272 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:51,274 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:51,554 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:51,554 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:52,164 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:52,169 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:52,169 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:52,170 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:52,555 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:53,067 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:53,069 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:53,070 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:53,071 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:53,557 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:53,557 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:53,952 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:53,955 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:53,955 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:53,955 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:53,956 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:54,852 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:54,854 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:54,855 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:54,856 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:55,560 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:55,560 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:55,746 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:55,749 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:55,749 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:55,750 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:56,561 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:56,754 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:56,757 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:56,757 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:56,758 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:57,562 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:57,563 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:57,628 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:57,630 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:57,630 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:57,631 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:58,522 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:58,525 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:58,525 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:58,527 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:58,550 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:49:58,564 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:49:58,643 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:49:58,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:49:58,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:49:59,486 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:49:59,488 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:49:59,488 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:49:59,489 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:49:59,490 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:49:59,565 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:49:59,565 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:00,369 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:00,372 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:00,372 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:00,374 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:00,566 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:01,266 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:01,269 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:01,269 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:01,270 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:01,567 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:01,567 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:02,139 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:02,141 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:02,142 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:02,143 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:02,568 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:03,023 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:03,025 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:03,026 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:03,027 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:03,569 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:03,570 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:03,892 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:03,895 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:03,896 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:03,897 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:04,776 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:04,779 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:04,779 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:04,779 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:04,780 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:05,572 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:05,572 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:05,761 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:05,764 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:05,764 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:05,765 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:06,573 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:06,627 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:06,630 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:06,630 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:06,631 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:07,499 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:07,502 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:07,503 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:07,504 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:07,575 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:07,575 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:08,398 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:08,401 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:08,401 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:08,402 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:08,576 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:09,270 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:09,272 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:09,273 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:09,274 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:09,578 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:09,578 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:10,168 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:10,170 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:10,171 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:10,171 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:10,172 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:10,579 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:11,040 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:11,043 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:11,043 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:11,044 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:11,580 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:11,580 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:11,905 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:11,908 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:11,908 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:11,909 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:12,780 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:12,782 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:12,783 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:12,784 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:13,583 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:13,583 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:13,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:50:13,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:50:13,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:50:13,750 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:13,904 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:13,905 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:13,906 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:14,584 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:14,661 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:14,664 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:14,664 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:14,665 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:15,546 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:15,548 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:15,549 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:15,549 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:15,550 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:15,585 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:15,585 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:16,423 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:16,426 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:16,426 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:16,427 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:16,586 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:17,303 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:17,306 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:17,307 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:17,308 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:17,587 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:17,588 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:18,189 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:18,191 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:18,192 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:18,193 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:18,589 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:19,095 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:19,098 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:19,098 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:19,099 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:19,590 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:19,590 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:19,982 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:19,984 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:19,985 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:19,986 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:20,880 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:20,883 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:20,883 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:20,884 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:20,885 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:21,592 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:21,593 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:21,851 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:21,854 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:21,855 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:21,856 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:22,594 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:23,594 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:25,595 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:26,419 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:27,348 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:27,597 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:28,399 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:28,401 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:28,402 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:28,403 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:28,551 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:50:28,598 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:28,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:50:28,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:50:28,661 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:50:29,275 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:29,277 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:29,277 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:29,278 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:29,599 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:29,599 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:30,158 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:30,160 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:30,161 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:30,162 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:30,600 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:31,053 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:31,056 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:31,056 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:31,058 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:31,601 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:31,601 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:31,954 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:31,956 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:31,957 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:31,957 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:31,958 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:32,848 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:32,851 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:32,851 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:32,852 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:33,603 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:33,604 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:33,725 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:33,727 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:33,728 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:33,729 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:34,605 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:34,613 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:34,616 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:34,616 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:34,617 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:35,488 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:35,490 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:35,491 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:35,492 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:35,606 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:35,606 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:36,476 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:36,478 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:36,478 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:36,480 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:36,607 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:37,372 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:37,374 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:37,375 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:37,375 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:37,376 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:37,608 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:37,608 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:38,247 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:38,250 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:38,250 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:38,251 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:38,609 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:39,133 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:39,136 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:39,136 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:39,137 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:39,610 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:39,611 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:40,013 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:40,016 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:40,016 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:40,018 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:40,612 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:41,157 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:41,160 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:41,161 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:41,162 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:41,613 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:41,613 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:42,050 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:42,053 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:42,053 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:42,054 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:42,614 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:42,935 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:42,937 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:42,938 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:42,938 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:42,939 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:43,615 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:43,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:50:43,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:50:43,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:50:43,828 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:43,853 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:43,854 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:43,855 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:44,617 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:44,804 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:44,806 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:44,806 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:44,807 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:45,618 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:45,618 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:45,689 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:45,691 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:45,692 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:45,693 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:46,577 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:46,579 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:46,580 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:46,581 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:46,619 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:47,443 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:47,445 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:47,446 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:47,447 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:47,620 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:47,620 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:48,318 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:48,321 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:48,321 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:48,322 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:48,323 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:48,621 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:49,282 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:49,285 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:49,285 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:49,286 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:49,623 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:49,623 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:50,170 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:50,173 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:50,173 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:50,174 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:50,624 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:51,057 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:51,060 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:51,060 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:51,062 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:51,625 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:51,625 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:51,919 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:51,945 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:51,946 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:51,947 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:52,822 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:52,824 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:52,825 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:52,826 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:53,628 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:53,628 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:53,819 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:53,822 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:53,822 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:53,823 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:53,824 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:54,629 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:54,692 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:54,695 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:54,695 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:54,696 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:55,575 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:55,578 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:55,578 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:55,580 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:55,630 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:55,630 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:56,432 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:56,434 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:56,435 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:56,436 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:56,631 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:57,281 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:57,283 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:57,284 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:57,285 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:57,632 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:57,633 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:58,166 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:58,169 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:58,169 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:58,171 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:58,552 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:50:58,634 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:58,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:50:58,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:50:58,661 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:50:58,894 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:50:59,047 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:59,049 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:59,050 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:59,051 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:50:59,635 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:50:59,635 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:50:59,920 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:50:59,923 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:50:59,923 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:50:59,924 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:00,820 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:00,823 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:00,823 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:00,825 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:01,637 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:01,637 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:01,891 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:01,894 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:01,894 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:01,895 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:02,638 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:02,777 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:02,780 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:02,780 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:02,781 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:03,640 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:03,642 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:03,643 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:03,643 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:03,643 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:03,644 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:04,504 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:04,507 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:04,507 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:04,507 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:04,508 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:04,644 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:05,387 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:05,390 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:05,391 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:05,392 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:05,645 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:05,646 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:06,267 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:06,270 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:06,270 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:06,271 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:06,647 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:07,159 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:07,161 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:07,161 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:07,163 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:07,648 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:07,648 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:08,050 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:08,052 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:08,053 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:08,054 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:08,649 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:08,931 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:08,934 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:08,934 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:08,935 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:09,650 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:09,818 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:09,821 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:09,822 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:09,822 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:09,823 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:10,651 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:10,828 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:10,830 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:10,831 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:10,832 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:11,653 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:11,653 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:11,710 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:11,712 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:11,712 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:11,714 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:12,589 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:12,592 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:12,592 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:12,594 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:12,654 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:13,466 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:13,468 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:13,469 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:13,470 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:13,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:51:13,644 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:51:13,655 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:13,655 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:13,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:51:14,343 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:14,345 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:14,346 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:14,347 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:14,656 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:15,216 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:15,219 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:15,219 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:15,219 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:15,221 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:15,657 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:15,657 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:16,102 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:16,106 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:16,106 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:16,107 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:16,658 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:17,080 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:17,083 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:17,084 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:17,085 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:17,659 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:17,660 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:17,958 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:17,960 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:17,961 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:17,962 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:18,964 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:18,967 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:18,968 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:18,969 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:19,662 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:19,662 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:19,852 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:19,855 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:19,855 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:19,856 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:20,663 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:20,719 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:20,721 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:20,722 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:20,722 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:20,723 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:21,664 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:21,664 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:21,689 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:21,691 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:21,692 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:21,693 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:22,548 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:22,551 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:22,551 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:22,552 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:22,665 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:23,436 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:23,438 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:23,438 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:23,440 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:23,666 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:23,667 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:24,322 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:24,324 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:24,325 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:24,326 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:24,668 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:25,207 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:25,209 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:25,210 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:25,211 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:25,669 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:25,669 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:26,093 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:26,096 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:26,097 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:26,097 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:26,098 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:26,670 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:27,082 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:27,085 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:27,085 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:27,087 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:27,671 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:27,672 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:27,976 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:27,979 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:27,979 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:27,980 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:28,553 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:51:28,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:51:28,645 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:51:28,661 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:51:28,879 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:28,881 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:28,882 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:28,883 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:29,674 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:29,674 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:29,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:29,771 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:29,772 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:29,773 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:30,661 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:30,664 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:30,664 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:30,665 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:30,675 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:31,545 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:31,548 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:31,548 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:31,548 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:31,550 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:31,676 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:31,676 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:32,446 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:32,449 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:32,449 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:32,450 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:32,677 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:33,323 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:33,326 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:33,326 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:33,327 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:33,678 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:33,678 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:34,208 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:34,210 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:34,210 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:34,211 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:34,679 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:35,184 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:35,186 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:35,187 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:35,190 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:35,681 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:35,681 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:36,066 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:36,068 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:36,069 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:36,070 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:36,682 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:36,963 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:36,966 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:36,966 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:36,966 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:36,967 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:37,683 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:37,836 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:37,838 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:37,839 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:37,840 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:38,684 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:38,724 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:38,726 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:38,727 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:38,728 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:39,683 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:39,685 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:39,686 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:39,687 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:39,687 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:39,687 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:40,570 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:40,572 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:40,572 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:40,574 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:40,688 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:41,438 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:41,444 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:41,444 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:41,445 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:41,689 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:41,689 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:42,288 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:42,290 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:42,291 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:42,291 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:42,293 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:42,690 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:43,175 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:43,177 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:43,178 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:43,179 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:43,692 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:43,692 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:43,755 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:51:43,756 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:51:43,756 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:51:44,210 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:44,212 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:44,213 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:44,214 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:44,693 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:45,081 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:45,084 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:45,084 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:45,085 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:45,694 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:45,695 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:45,968 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:45,970 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:45,970 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:45,972 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:46,864 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:46,867 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:46,869 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:46,870 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:47,697 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:47,697 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:47,751 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:47,753 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:47,754 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:47,754 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:47,755 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:48,644 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:48,648 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:48,648 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:48,649 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:48,698 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:49,532 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:49,535 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:49,535 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:49,536 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:49,699 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:49,699 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:50,445 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:50,448 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:50,448 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:50,449 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:50,700 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:51,323 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:51,326 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:51,326 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:51,328 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:51,701 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:51,702 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:52,331 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:52,334 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:52,334 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:52,335 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:52,703 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:53,219 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:53,221 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:53,222 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:53,222 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:53,223 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:53,704 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:53,704 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:54,127 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:54,129 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:54,130 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:54,134 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:54,705 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:55,024 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:55,026 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:55,027 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:55,028 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:55,706 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:55,706 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:55,936 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:55,939 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:55,939 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:55,940 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:56,904 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:56,909 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:56,909 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:56,910 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:57,709 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:51:57,709 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:57,905 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:51:57,908 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:51:57,908 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:51:57,910 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:51:58,554 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:51:58,554 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:51:58,710 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:51:58,776 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:51:58,777 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:51:58,817 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:51:59,711 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:01,712 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:02,952 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:03,714 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:03,852 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:03,855 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:03,856 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:03,856 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:03,857 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:04,715 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:04,758 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:04,761 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:04,761 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:04,762 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:05,716 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:05,716 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:05,837 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:05,839 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:05,840 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:05,841 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:06,717 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:06,744 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:06,747 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:06,747 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:06,748 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:07,650 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:07,652 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:07,653 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:07,655 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:07,719 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:07,719 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:08,552 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:08,555 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:08,556 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:08,557 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:08,720 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:09,423 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:09,425 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:09,426 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:09,426 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:09,427 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:09,721 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:09,721 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:10,300 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:10,303 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:10,303 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:10,304 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:10,722 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:11,169 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:11,171 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:11,172 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:11,173 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:11,723 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:11,723 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:12,059 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:12,062 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:12,062 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:12,063 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:12,724 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:12,921 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:12,923 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:12,924 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:12,925 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:13,725 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:13,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:52:13,764 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:52:13,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:52:13,901 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:14,018 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:14,018 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:14,019 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:14,726 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:14,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:14,778 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:14,778 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:14,778 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:14,780 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:15,656 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:15,658 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:15,658 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:15,659 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:15,727 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:15,727 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:16,807 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:16,810 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:16,810 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:16,811 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:17,698 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:17,701 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:17,701 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:17,702 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:17,729 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:17,729 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:18,558 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:18,560 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:18,560 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:18,561 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:18,730 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:19,414 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:19,416 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:19,417 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:19,418 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:19,731 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:19,731 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:20,273 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:20,276 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:20,276 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:20,276 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:20,277 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:20,732 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:21,152 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:21,154 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:21,155 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:21,156 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:21,733 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:21,733 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:22,051 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:22,053 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:22,053 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:22,054 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:22,734 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:23,049 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:23,052 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:23,052 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:23,053 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:23,735 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:23,735 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:23,917 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:23,919 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:23,920 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:23,921 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:24,811 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:24,814 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:24,814 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:24,815 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:25,690 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:25,693 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:25,693 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:25,693 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:25,694 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:25,737 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:25,737 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:26,547 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:26,550 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:26,550 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:26,551 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:26,738 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:27,517 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:27,548 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:27,549 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:27,550 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:27,739 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:27,739 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:28,399 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:28,401 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:28,402 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:28,403 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:28,554 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:52:28,740 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:28,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:52:28,764 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:52:28,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:52:29,282 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:29,284 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:29,284 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:29,285 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:29,741 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:29,742 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:30,167 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:30,169 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:30,170 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:30,171 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:30,743 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:31,144 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:31,147 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:31,147 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:31,147 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:31,148 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:31,744 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:31,744 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:31,988 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:31,990 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:31,991 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:31,992 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:32,837 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:32,840 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:32,840 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:32,841 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:33,728 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:33,730 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:33,730 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:33,731 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:33,746 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:33,746 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:34,588 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:34,590 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:34,591 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:34,592 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:34,747 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:35,483 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:35,485 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:35,485 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:35,487 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:35,748 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:35,748 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:36,360 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:36,363 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:36,363 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:36,363 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:36,364 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:36,749 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:37,253 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:37,255 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:37,256 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:37,257 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:37,749 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:37,750 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:38,127 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:38,130 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:38,130 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:38,131 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:38,750 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:38,980 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:38,982 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:38,982 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:38,983 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:39,752 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:39,971 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:39,973 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:39,973 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:39,975 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:40,753 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:40,855 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:40,858 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:40,858 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:40,859 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:41,735 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:41,738 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:41,738 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:41,739 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:41,739 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:41,754 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:41,754 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:42,627 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:42,629 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:42,629 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:42,631 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:42,755 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:43,516 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:43,518 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:43,518 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:43,519 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:43,756 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:43,757 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:43,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:52:43,764 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:52:43,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:52:44,409 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:44,410 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:44,411 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:44,412 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:44,757 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:45,301 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:45,303 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:45,304 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:45,305 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:45,759 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:45,759 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:46,194 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:46,197 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:46,197 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:46,198 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:46,760 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:47,169 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:47,172 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:47,172 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:47,172 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:47,173 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:47,761 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:47,761 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:48,049 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:48,052 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:48,052 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:48,053 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:48,762 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:48,938 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:48,940 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:48,940 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:48,943 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:49,764 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:49,861 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:49,863 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:49,864 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:49,865 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:50,750 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:50,753 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:50,753 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:50,755 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:50,765 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:51,648 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:51,651 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:51,651 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:51,653 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:51,766 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:51,766 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:52,520 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:52,523 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:52,524 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:52,524 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:52,525 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:52,767 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:53,411 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:53,413 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:53,414 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:53,415 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:53,768 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:53,768 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:54,283 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:54,286 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:54,286 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:54,287 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:54,769 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:55,173 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:55,176 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:55,176 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:55,177 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:55,771 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:55,771 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:56,177 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:56,179 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:56,179 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:56,181 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:56,772 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:57,051 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:57,055 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:57,055 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:57,056 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:57,773 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:57,773 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:52:57,925 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:57,927 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:57,928 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:57,928 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:52:57,929 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:58,556 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:52:58,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:52:58,764 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:52:58,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:52:58,798 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:59,019 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:59,019 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:59,021 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:59,688 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:52:59,691 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:52:59,691 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:52:59,692 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:52:59,775 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:52:59,776 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:00,576 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:00,578 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:00,578 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:00,580 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:00,777 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:01,455 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:01,458 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:01,458 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:01,459 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:01,778 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:01,778 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:02,351 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:02,354 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:02,354 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:02,357 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:02,779 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:03,258 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:03,261 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:03,262 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:03,262 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:03,263 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:03,780 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:03,780 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:04,268 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:04,271 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:04,271 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:04,272 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:04,782 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:05,174 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:05,177 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:05,177 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:05,178 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:05,783 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:05,783 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:06,057 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:06,059 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:06,060 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:06,061 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:06,784 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:06,960 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:06,962 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:06,962 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:06,964 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:07,785 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:07,840 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:07,842 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:07,843 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:07,844 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:08,729 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:08,732 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:08,732 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:08,733 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:08,734 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:08,787 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:09,639 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:09,642 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:09,642 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:09,643 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:09,788 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:09,788 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:10,519 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:10,522 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:10,522 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:10,524 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:10,789 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:11,434 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:11,437 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:11,437 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:11,438 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:11,790 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:11,790 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:12,307 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:12,310 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:12,310 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:12,312 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:12,791 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:13,307 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:13,310 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:13,310 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:13,311 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:13,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:53:13,764 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:53:13,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:53:13,793 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:13,793 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:13,987 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:14,181 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:14,182 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:14,183 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:14,184 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:14,794 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:15,064 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:15,067 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:15,067 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:15,069 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:15,795 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:15,795 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:15,980 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:15,983 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:15,983 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:15,984 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:16,845 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:16,847 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:16,848 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:16,849 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:17,724 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:17,726 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:17,726 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:17,728 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:17,797 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:17,797 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:18,631 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:18,633 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:18,634 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:18,635 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:18,798 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:19,498 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:19,501 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:19,501 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:19,501 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:19,503 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:19,800 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:19,800 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:20,396 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:20,398 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:20,398 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:20,400 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:20,801 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:21,478 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:21,480 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:21,481 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:21,482 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:21,802 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:21,802 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:22,346 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:22,348 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:22,349 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:22,350 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:22,803 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:23,222 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:23,225 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:23,225 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:23,226 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:23,804 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:23,805 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:24,091 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:24,094 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:24,094 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:24,095 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:24,806 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:24,990 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:24,992 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:24,993 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:24,993 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:24,994 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:25,807 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:25,883 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:25,886 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:25,886 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:25,887 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:26,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:26,769 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:26,770 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:26,771 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:26,808 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:27,649 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:27,652 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:27,652 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:27,654 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:27,809 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:27,810 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:28,537 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:28,539 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:28,540 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:28,541 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:28,557 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:53:28,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:53:28,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:53:28,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:53:28,810 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:29,445 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:29,446 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:29,447 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:29,448 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:29,812 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:29,812 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:30,449 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:30,451 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:30,452 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:30,452 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:30,454 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:30,813 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:31,348 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:31,350 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:31,351 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:31,353 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:31,814 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:31,815 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:32,334 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:32,336 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:32,337 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:32,338 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:32,816 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:33,221 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:33,224 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:33,225 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:33,226 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:33,817 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:33,817 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:35,761 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:35,818 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:37,819 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:38,412 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:39,314 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:39,317 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:39,317 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:39,318 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:39,821 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:39,821 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:40,203 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:40,205 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:40,206 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:40,207 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:40,822 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:41,102 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:41,104 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:41,105 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:41,105 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:41,106 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:41,824 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:41,824 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:42,011 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:42,013 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:42,014 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:42,015 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:42,825 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:42,909 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:42,912 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:42,912 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:42,913 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:43,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:53:43,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:53:43,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:53:43,826 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:43,950 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:43,971 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:43,972 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:43,973 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:44,827 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:44,839 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:44,841 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:44,841 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:44,842 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:45,717 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:45,719 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:45,719 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:45,721 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:45,829 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:45,829 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:46,607 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:46,609 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:46,610 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:46,610 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:46,611 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:46,830 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:47,505 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:47,508 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:47,508 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:47,510 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:47,831 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:47,831 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:48,388 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:48,392 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:48,392 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:48,393 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:48,832 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:49,274 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:49,276 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:49,277 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:49,278 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:49,833 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:49,833 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:50,158 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:50,160 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:50,161 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:50,162 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:50,834 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:51,430 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:51,433 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:51,433 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:51,434 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:51,836 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:51,836 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:52,321 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:52,323 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:52,324 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:52,324 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:52,325 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:52,837 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:53,213 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:53,215 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:53,216 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:53,217 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:53,838 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:53,838 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:54,074 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:54,076 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:54,077 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:54,078 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:54,839 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:54,931 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:54,933 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:54,933 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:54,935 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:55,810 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:55,815 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:55,815 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:55,817 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:55,841 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:55,841 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:56,704 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:56,707 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:56,707 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:56,708 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:56,842 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:57,604 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:57,606 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:57,607 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:57,607 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:53:57,608 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:57,843 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:57,843 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:58,479 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:58,481 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:58,482 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:58,483 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:58,557 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:53:58,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:53:58,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:53:58,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:53:58,844 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:53:59,377 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:53:59,379 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:53:59,379 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:53:59,380 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:53:59,845 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:53:59,845 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:00,268 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:00,270 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:00,270 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:00,272 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:00,846 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:01,279 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:01,282 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:01,282 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:01,284 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:01,847 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:01,848 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:02,145 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:02,147 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:02,148 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:02,149 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:02,849 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:03,048 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:03,051 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:03,051 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:03,052 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:03,053 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:03,850 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:03,850 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:03,941 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:03,943 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:03,944 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:03,945 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:04,819 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:04,823 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:04,823 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:04,824 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:04,851 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:05,719 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:05,721 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:05,722 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:05,723 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:05,852 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:05,852 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:06,583 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:06,585 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:06,586 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:06,587 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:06,853 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:07,434 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:07,437 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:07,437 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:07,438 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:07,854 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:07,855 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:08,439 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:08,442 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:08,471 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:08,472 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:08,473 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:08,855 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:09,332 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:09,335 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:09,335 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:09,336 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:09,857 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:09,857 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:10,204 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:10,206 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:10,207 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:10,208 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:10,858 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:11,085 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:11,088 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:11,088 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:11,089 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:11,859 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:11,859 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:11,969 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:11,972 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:11,972 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:11,974 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:12,859 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:12,862 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:12,862 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:12,863 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:12,864 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:13,732 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:13,735 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:13,735 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:13,735 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:13,736 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:13,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:54:13,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:54:13,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:54:13,865 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:13,865 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:14,675 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:14,677 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:14,678 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:14,679 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:14,866 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:15,566 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:15,569 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:15,569 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:15,570 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:15,867 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:15,867 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:16,454 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:16,456 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:16,457 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:16,458 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:16,868 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:17,457 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:17,460 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:17,460 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:17,461 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:17,869 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:17,869 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:18,355 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:18,358 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:18,358 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:18,359 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:18,870 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:19,229 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:19,231 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:19,232 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:19,232 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:19,233 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:19,872 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:19,872 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:20,114 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:20,116 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:20,117 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:20,118 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:20,873 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:21,015 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:21,018 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:21,018 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:21,020 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:21,874 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:21,874 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:21,902 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:21,905 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:21,905 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:21,906 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:22,783 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:22,786 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:22,786 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:22,788 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:22,875 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:23,659 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:23,661 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:23,662 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:23,663 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:23,876 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:23,876 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:24,537 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:24,540 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:24,540 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:24,540 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:24,541 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:24,877 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:25,520 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:25,523 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:25,523 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:25,524 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:25,879 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:25,879 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:26,409 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:26,412 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:26,412 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:26,413 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:26,881 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:27,311 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:27,313 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:27,313 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:27,315 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:27,882 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:27,882 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:28,220 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:28,223 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:28,223 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:28,225 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:28,558 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:54:28,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:54:28,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:54:28,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:54:28,883 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:29,099 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:29,101 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:29,101 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:29,102 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:29,884 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:29,885 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:29,997 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:30,000 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:30,000 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:30,000 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:30,001 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:30,883 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:30,885 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:30,886 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:30,887 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:30,887 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:31,776 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:31,779 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:31,779 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:31,780 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:31,889 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:31,889 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:32,640 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:32,642 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:32,643 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:32,644 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:32,889 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:33,523 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:33,526 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:33,526 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:33,527 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:33,891 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:33,891 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:34,530 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:34,532 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:34,532 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:34,534 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:34,892 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:35,410 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:35,413 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:35,413 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:35,413 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:35,415 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:35,893 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:35,893 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:36,291 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:36,294 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:36,294 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:36,295 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:36,894 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:37,180 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:37,183 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:37,184 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:37,185 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:37,895 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:37,896 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:38,101 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:38,104 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:38,104 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:38,105 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:38,897 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:39,003 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:39,006 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:39,007 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:39,008 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:39,897 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:39,900 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:39,900 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:39,902 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:39,902 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:39,902 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:40,795 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:40,798 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:40,799 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:40,799 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:40,800 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:40,903 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:41,802 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:41,804 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:41,804 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:41,805 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:41,904 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:41,904 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:42,708 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:42,711 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:42,711 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:42,713 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:42,905 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:43,593 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:43,596 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:43,596 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:43,597 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:43,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:54:43,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:54:43,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:54:43,906 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:43,907 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:44,484 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:44,485 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:44,486 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:44,487 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:44,907 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:45,367 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:45,370 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:45,370 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:45,371 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:45,909 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:45,909 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:46,258 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:46,261 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:46,262 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:46,262 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:46,263 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:46,910 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:47,138 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:47,140 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:47,141 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:47,142 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:47,911 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:47,911 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:48,016 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:48,019 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:48,019 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:48,020 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:48,911 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:48,914 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:48,914 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:48,915 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:48,915 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:49,788 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:49,790 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:49,791 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:49,792 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:49,916 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:49,916 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:50,797 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:50,800 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:50,801 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:50,802 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:50,917 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:51,683 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:51,686 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:51,687 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:51,687 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:51,688 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:51,918 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:51,918 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:52,549 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:52,551 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:52,551 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:52,553 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:52,919 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:53,441 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:53,445 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:53,446 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:53,447 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:53,921 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:53,921 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:54,336 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:54,338 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:54,339 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:54,340 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:54,922 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:55,303 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:55,305 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:55,305 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:55,308 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:55,923 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:55,923 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:56,187 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:56,190 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:56,190 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:56,191 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:56,924 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:57,073 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:57,076 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:57,077 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:57,077 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:54:57,078 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:57,925 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:57,926 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:54:57,941 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:57,944 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:57,944 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:57,945 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:58,559 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:54:58,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:54:58,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:54:58,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:54:58,927 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:58,985 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:58,986 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:58,987 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:59,811 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:54:59,813 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:54:59,813 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:54:59,814 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:54:59,927 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:54:59,928 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:00,717 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:00,719 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:00,720 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:00,721 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:00,929 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:01,590 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:01,593 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:01,593 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:01,594 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:01,930 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:01,930 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:02,480 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:02,482 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:02,483 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:02,483 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:02,484 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:02,931 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:03,368 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:03,371 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:03,371 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:03,373 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:03,932 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:03,932 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:04,278 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:04,280 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:04,281 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:04,282 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:04,933 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:05,174 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:05,176 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:05,177 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:05,178 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:05,934 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:05,935 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:06,080 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:06,082 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:06,083 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:06,084 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:06,936 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:06,966 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:06,969 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:06,969 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:06,971 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:07,937 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:07,971 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:08,109 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:08,112 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:08,112 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:08,114 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:08,938 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:08,991 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:08,993 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:08,994 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:08,995 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:09,939 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:11,940 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:13,289 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:13,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:55:13,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:55:13,809 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:55:13,942 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:14,065 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:14,925 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:14,928 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:14,928 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:14,929 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:14,943 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:15,818 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:15,821 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:15,822 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:15,823 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:15,944 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:15,944 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:16,718 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:16,720 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:16,721 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:16,722 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:16,945 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:17,693 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:17,695 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:17,696 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:17,697 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:17,946 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:17,946 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:18,582 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:18,585 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:18,585 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:18,585 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:18,587 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:18,947 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:19,454 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:19,457 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:19,457 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:19,458 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:19,948 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:19,949 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:20,333 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:20,336 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:20,336 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:20,337 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:20,950 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:21,368 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:21,371 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:21,371 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:21,372 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:21,951 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:21,951 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:22,263 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:22,266 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:22,266 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:22,268 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:22,952 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:23,134 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:23,136 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:23,136 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:23,138 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:23,953 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:23,953 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:24,023 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:24,025 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:24,026 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:24,026 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:24,027 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:24,906 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:24,908 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:24,909 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:24,911 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:24,954 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:25,797 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:25,799 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:25,801 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:25,802 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:25,955 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:25,956 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:26,975 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:26,977 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:26,978 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:26,979 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:27,871 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:27,874 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:27,875 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:27,876 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:27,957 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:27,957 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:28,560 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:55:28,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:55:28,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:55:28,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:55:28,851 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:29,032 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:29,033 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:29,033 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:29,034 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:29,727 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:29,729 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:29,730 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:29,731 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:29,959 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:29,959 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:30,596 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:30,599 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:30,600 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:30,601 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:30,960 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:31,478 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:31,481 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:31,482 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:31,483 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:31,961 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:31,962 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:32,380 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:32,384 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:32,385 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:32,386 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:32,963 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:33,247 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:33,250 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:33,250 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:33,252 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:33,964 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:33,964 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:34,159 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:34,161 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:34,162 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:34,162 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:34,163 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:34,965 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:35,059 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:35,062 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:35,062 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:35,063 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:35,947 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:35,953 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:35,954 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:35,955 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:35,966 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:35,966 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:36,846 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:36,848 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:36,849 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:36,851 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:36,967 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:37,857 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:37,860 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:37,860 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:37,861 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:37,969 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:37,969 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:38,750 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:38,753 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:38,754 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:38,755 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:38,970 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:39,646 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:39,648 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:39,649 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:39,649 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:39,651 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:39,971 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:39,971 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:40,541 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:40,543 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:40,544 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:40,545 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:40,972 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:41,409 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:41,411 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:41,412 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:41,413 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:41,973 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:41,974 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:42,259 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:42,262 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:42,262 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:42,263 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:42,975 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:43,126 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:43,129 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:43,130 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:43,131 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:43,765 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:55:43,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:55:43,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:55:43,976 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:43,976 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:44,033 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:44,035 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:44,036 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:44,037 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:44,911 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:44,937 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:44,938 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:44,938 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:44,939 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:44,977 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:45,914 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:45,917 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:45,918 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:45,919 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:45,978 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:45,978 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:46,780 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:46,782 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:46,783 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:46,784 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:46,979 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:47,679 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:47,682 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:47,682 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:47,683 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:47,981 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:47,981 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:48,567 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:48,570 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:48,570 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:48,571 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:48,982 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:49,444 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:49,446 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:49,447 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:49,448 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:49,983 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:49,983 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:50,331 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:50,334 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:50,335 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:50,335 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:50,336 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:50,984 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:51,297 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:51,299 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:51,300 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:51,301 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:51,985 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:51,986 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:52,189 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:52,191 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:52,191 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:52,193 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:52,986 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:53,081 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:53,084 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:53,085 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:53,086 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:53,969 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:53,972 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:53,972 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:53,975 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:53,987 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:53,988 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:54,971 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:54,974 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:54,974 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:54,975 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:54,989 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:55,863 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:55,865 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:55,866 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:55,866 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:55:55,867 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:55,990 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:55,990 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:56,757 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:56,760 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:56,760 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:56,761 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:56,991 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:57,636 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:57,639 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:57,639 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:57,640 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:57,992 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:57,992 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:58,516 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:58,520 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:58,520 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:58,521 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:58,561 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:55:58,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:55:58,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:55:58,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:55:58,993 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:55:59,410 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:55:59,412 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:55:59,412 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:55:59,414 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:55:59,994 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:55:59,994 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:00,281 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:00,284 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:00,284 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:00,285 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:00,995 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:01,168 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:01,170 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:01,170 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:01,170 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:01,172 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:01,996 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:01,997 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:02,058 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:02,061 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:02,061 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:02,062 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:02,998 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:03,040 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:03,043 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:03,043 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:03,045 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:03,936 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:03,939 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:03,939 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:03,940 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:03,999 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:03,999 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:04,817 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:04,819 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:04,820 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:04,821 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:05,000 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:05,728 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:05,731 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:05,731 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:05,732 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:06,001 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:06,001 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:06,624 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:06,626 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:06,626 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:06,627 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:06,628 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:07,002 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:07,497 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:07,499 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:07,500 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:07,501 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:08,003 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:08,004 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:08,379 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:08,382 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:08,382 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:08,383 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:09,004 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:09,260 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:09,263 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:09,263 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:09,265 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:10,006 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:10,006 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:10,149 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:10,151 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:10,152 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:10,153 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:11,007 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:11,028 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:11,031 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:11,031 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:11,032 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:12,001 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:12,006 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:12,006 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:12,007 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:12,007 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:12,008 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:12,008 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:12,897 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:12,899 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:12,900 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:12,902 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:13,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:56:13,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:56:13,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:56:13,800 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:14,010 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:14,019 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:14,020 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:14,021 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:14,709 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:14,712 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:14,713 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:14,714 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:15,011 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:15,673 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:15,675 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:15,675 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:15,677 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:16,012 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:16,012 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:16,561 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:16,564 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:16,564 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:16,565 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:17,013 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:17,450 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:17,455 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:17,455 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:17,456 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:17,457 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:18,016 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:18,016 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:18,315 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:18,319 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:18,319 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:18,320 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:19,017 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:19,313 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:19,315 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:19,316 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:19,317 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:20,019 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:20,019 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:20,196 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:20,199 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:20,199 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:20,200 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:21,020 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:21,073 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:21,076 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:21,076 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:21,077 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:21,971 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:21,975 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:21,975 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:21,976 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:22,022 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:22,022 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:22,835 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:22,837 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:22,838 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:22,838 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:22,839 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:23,023 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:23,712 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:23,715 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:23,715 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:23,716 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:24,025 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:24,025 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:24,580 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:24,582 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:24,582 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:24,583 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:25,026 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:25,466 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:25,468 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:25,469 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:25,470 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:26,029 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:26,030 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:26,353 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:26,356 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:26,356 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:26,358 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:27,030 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:27,220 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:27,222 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:27,223 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:27,224 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:28,033 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:28,033 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:28,195 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:28,199 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:28,200 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:28,200 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:28,201 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:28,562 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:56:28,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:56:28,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:56:28,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:56:29,034 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:29,092 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:29,093 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:29,094 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:29,095 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:29,965 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:29,967 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:29,968 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:29,969 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:30,035 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:30,035 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:30,870 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:30,873 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:30,873 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:30,881 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:31,036 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:31,784 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:31,787 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:31,787 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:31,788 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:32,038 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:32,038 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:32,670 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:32,673 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:32,673 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:32,675 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:33,039 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:33,559 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:33,562 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:33,562 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:33,562 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:33,563 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:34,041 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:34,042 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:34,449 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:34,451 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:34,453 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:34,454 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:35,043 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:35,355 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:35,357 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:35,358 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:35,359 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:36,044 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:36,045 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:36,367 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:36,369 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:36,370 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:36,371 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:37,046 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:37,256 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:37,258 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:37,259 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:37,260 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:38,048 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:38,048 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:38,172 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:38,175 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:38,175 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:38,177 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:39,049 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:39,076 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:39,079 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:39,079 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:39,080 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:39,081 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:39,987 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:39,989 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:39,990 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:39,991 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:40,051 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:40,051 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:40,891 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:40,894 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:40,894 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:40,896 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:41,052 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:41,801 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:41,803 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:41,804 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:41,805 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:42,054 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:42,054 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:42,795 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:42,797 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:42,798 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:42,799 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:43,055 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:43,695 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:43,698 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:43,698 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:43,699 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:43,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:56:43,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:56:43,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:56:44,057 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:44,057 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:44,590 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:44,592 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:44,592 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:44,592 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:44,593 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:45,058 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:46,060 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:48,061 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:49,062 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:49,642 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:49,643 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:50,556 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:50,558 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:50,558 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:50,560 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:51,064 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:51,064 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:51,452 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:51,454 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:51,455 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:51,456 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:52,065 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:52,348 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:52,350 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:52,350 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:52,352 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:53,066 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:53,066 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:53,235 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:53,237 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:53,237 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:53,239 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:54,067 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:54,104 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:54,106 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:54,106 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:54,107 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:54,981 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:54,983 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:54,983 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:54,984 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:56:54,985 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:55,068 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:55,068 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:55,848 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:55,850 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:55,851 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:55,852 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:56,069 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:56,727 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:56,729 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:56,729 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:56,730 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:57,070 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:57,070 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:57,611 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:57,613 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:57,614 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:57,615 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:58,071 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:58,562 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:56:58,623 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:58,625 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:58,626 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:58,627 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:56:58,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:56:58,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:56:58,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:56:59,072 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:56:59,072 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:56:59,500 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:56:59,502 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:56:59,502 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:56:59,503 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:00,073 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:00,373 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:00,375 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:00,375 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:00,375 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:00,376 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:01,074 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:01,074 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:01,514 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:01,517 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:01,517 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:01,518 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:02,075 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:02,403 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:02,405 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:02,406 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:02,407 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:03,076 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:03,076 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:03,282 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:03,284 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:03,284 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:03,285 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:04,077 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:04,135 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:04,137 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:04,137 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:04,138 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:04,986 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:04,988 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:04,989 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:04,990 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:05,078 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:05,078 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:05,950 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:05,952 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:05,953 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:05,953 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:05,954 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:06,079 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:06,913 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:06,916 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:06,916 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:06,917 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:07,080 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:07,080 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:07,770 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:07,772 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:07,772 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:07,773 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:08,081 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:08,634 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:08,636 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:08,636 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:08,637 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:09,082 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:09,082 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:09,520 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:09,524 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:09,524 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:09,525 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:10,083 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:10,479 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:10,482 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:10,483 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:10,484 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:11,084 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:11,084 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:11,363 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:11,366 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:11,366 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:11,366 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:11,367 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:12,085 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:12,219 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:12,222 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:12,222 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:12,223 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:13,086 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:13,087 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:13,114 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:13,116 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:13,116 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:13,125 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:13,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:57:13,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:57:13,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:57:13,998 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:14,000 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:14,000 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:14,001 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:14,087 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:14,965 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:14,968 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:14,968 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:14,970 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:15,089 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:15,089 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:15,854 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:15,857 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:15,858 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:15,859 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:16,090 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:16,710 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:16,713 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:16,714 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:16,714 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:16,715 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:17,091 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:17,091 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:17,577 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:17,580 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:17,580 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:17,581 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:18,092 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:18,476 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:18,480 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:18,480 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:18,482 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:19,093 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:19,094 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:19,367 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:19,372 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:19,373 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:19,374 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:20,095 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:20,243 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:20,246 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:20,247 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:20,248 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:21,096 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:21,096 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:21,120 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:21,123 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:21,124 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:21,125 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:22,012 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:22,014 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:22,015 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:22,015 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:22,016 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:22,097 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:23,043 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:23,045 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:23,046 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:23,047 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:23,098 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:23,098 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:23,903 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:23,906 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:23,906 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:23,907 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:24,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:24,772 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:24,772 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:24,773 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:25,100 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:25,100 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:25,653 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:25,656 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:25,656 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:25,657 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:26,101 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:26,544 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:26,546 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:26,547 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:26,548 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:27,102 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:27,103 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:27,442 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:27,468 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:27,469 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:27,469 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:27,470 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:28,104 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:28,351 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:28,353 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:28,354 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:28,355 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:28,563 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:57:28,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:57:28,766 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:57:28,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:57:29,105 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:29,105 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:29,225 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:29,227 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:29,227 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:29,228 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:30,106 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:30,113 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:30,115 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:30,116 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:30,117 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:31,014 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:31,016 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:31,016 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:31,018 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:31,107 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:31,107 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:31,992 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:31,994 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:31,994 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:31,997 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:32,880 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:32,882 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:32,883 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:32,883 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:32,884 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:33,109 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:33,110 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:33,763 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:33,766 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:33,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:33,768 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:34,111 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:34,669 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:34,671 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:34,672 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:34,674 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:35,112 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:35,112 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:35,554 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:35,557 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:35,557 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:35,559 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:36,113 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:36,444 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:36,447 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:36,447 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:36,449 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:37,115 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:37,115 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:37,349 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:37,352 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:37,352 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:37,354 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:38,116 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:38,236 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:38,239 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:38,239 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:38,239 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:38,240 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:39,117 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:39,118 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:39,240 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:39,243 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:39,243 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:39,244 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:40,119 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:40,128 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:40,131 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:40,131 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:40,133 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:41,000 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:41,002 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:41,003 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:41,004 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:41,120 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:41,120 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:41,888 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:41,892 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:41,892 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:41,893 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:42,121 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:42,764 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:42,767 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:42,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:42,768 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:43,122 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:43,122 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:43,646 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:43,649 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:43,649 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:43,649 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:43,650 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:43,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:57:43,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:57:43,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:57:44,123 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:44,537 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:44,539 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:44,539 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:44,541 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:45,125 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:45,125 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:45,426 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:45,429 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:45,429 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:45,430 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:46,126 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:46,315 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:46,317 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:46,317 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:46,319 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:47,127 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:47,127 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:47,216 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:47,219 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:47,219 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:47,220 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:48,128 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:48,224 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:48,227 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:48,228 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:48,229 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:49,130 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:49,132 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:49,133 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:49,133 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:49,133 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:49,134 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:49,134 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:50,028 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:50,031 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:50,031 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:50,032 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:50,134 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:50,921 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:50,923 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:50,924 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:50,925 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:51,135 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:51,846 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:51,848 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:51,849 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:51,850 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:52,136 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:52,755 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:52,757 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:52,758 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:52,759 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:53,137 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:53,138 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:53,652 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:53,654 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:53,655 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:53,656 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:54,139 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:54,548 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:54,550 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:54,550 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:54,550 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:57:54,552 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:55,140 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:55,140 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:55,434 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:55,436 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:55,436 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:55,438 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:56,141 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:56,511 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:56,513 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:56,513 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:56,515 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:57,142 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:57,143 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:57,390 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:57,392 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:57,392 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:57,394 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:58,143 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:58,263 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:58,266 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:58,267 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:58,268 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:57:58,564 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:57:58,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:57:58,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:57:58,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:57:59,144 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:57:59,145 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:57:59,162 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:57:59,164 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:57:59,164 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:57:59,166 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:00,039 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:00,042 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:00,042 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:00,043 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:00,044 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:00,146 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:00,931 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:00,933 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:00,933 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:00,935 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:01,147 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:01,806 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:01,809 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:01,809 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:01,811 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:02,148 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:02,668 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:02,670 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:02,672 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:02,673 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:03,149 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:03,149 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:03,562 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:03,565 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:03,565 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:03,566 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:04,150 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:04,460 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:04,463 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:04,463 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:04,465 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:05,152 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:05,152 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:05,453 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:05,455 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:05,456 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:05,456 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:05,457 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:06,153 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:06,332 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:06,335 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:06,336 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:06,337 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:07,154 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:07,155 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:07,219 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:07,221 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:07,221 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:07,223 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:08,085 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:08,087 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:08,088 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:08,089 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:08,155 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:08,969 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:08,971 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:08,972 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:08,973 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:09,157 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:09,847 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:09,850 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:09,851 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:09,852 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:10,158 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:10,752 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:10,754 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:10,755 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:10,755 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:10,756 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:11,159 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:11,159 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:11,682 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:11,685 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:11,685 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:11,686 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:12,160 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:12,683 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:12,685 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:12,686 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:12,688 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:13,162 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:13,162 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:13,569 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:13,572 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:13,572 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:13,573 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:13,766 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:58:13,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:58:13,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:58:14,163 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:14,476 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:14,478 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:14,479 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:14,480 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:15,164 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:15,164 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:15,366 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:15,368 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:15,369 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:15,370 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:16,165 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:16,268 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:16,270 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:16,271 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:16,271 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:16,272 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:17,151 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:17,153 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:17,154 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:17,155 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:17,166 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:17,166 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:18,159 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:18,162 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:18,162 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:18,163 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:18,167 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:19,045 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:19,048 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:19,048 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:19,049 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:19,169 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:19,169 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:19,945 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:19,948 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:19,949 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:19,950 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:21,171 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:21,483 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:23,172 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:24,173 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:25,031 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:26,098 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:26,101 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:26,101 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:26,103 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:26,175 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:26,994 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:26,997 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:26,998 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:26,998 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:26,999 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:27,176 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:27,891 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:27,893 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:27,894 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:27,895 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:28,178 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:28,565 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:58:28,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:58:28,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:58:28,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:58:28,784 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:29,041 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:29,042 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:29,043 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:29,179 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:29,179 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:29,656 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:29,659 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:29,659 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:29,661 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:30,537 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:30,539 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:30,540 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:30,541 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:31,181 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:31,181 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:31,465 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:31,468 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:31,468 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:31,469 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:32,182 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:32,362 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:32,364 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:32,366 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:32,366 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:32,368 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:33,183 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:33,184 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:33,304 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:33,307 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:33,307 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:33,308 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:34,185 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:34,197 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:34,202 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:34,202 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:34,203 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:35,186 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:35,186 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:35,226 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:35,229 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:35,229 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:35,230 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:36,113 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:36,116 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:36,117 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:36,118 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:36,187 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:37,188 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:37,289 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:37,292 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:37,292 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:37,293 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:38,185 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:38,188 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:38,189 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:38,189 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:38,190 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:38,190 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:39,058 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:39,061 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:39,061 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:39,062 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:39,191 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:39,191 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:39,934 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:39,937 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:39,937 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:39,938 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:40,801 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:40,804 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:40,804 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:40,806 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:41,194 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:41,194 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:41,688 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:41,690 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:41,691 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:41,692 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:42,195 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:42,593 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:42,597 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:42,597 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:42,598 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:43,196 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:43,196 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:43,566 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:43,569 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:43,569 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:43,569 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:43,570 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:43,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:58:43,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:58:43,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:58:44,197 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:44,453 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:44,455 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:44,455 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:44,457 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:45,198 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:45,199 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:45,353 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:45,356 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:45,356 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:45,357 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:46,200 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:46,239 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:46,242 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:46,242 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:46,244 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:47,113 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:47,115 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:47,115 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:47,117 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:47,201 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:47,201 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:48,008 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:48,010 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:48,010 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:48,012 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:48,202 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:48,895 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:48,897 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:48,898 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:48,898 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:48,899 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:49,203 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:49,785 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:49,786 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:49,787 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:49,789 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:50,204 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:50,680 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:50,683 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:50,683 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:50,685 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:51,205 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:51,206 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:51,550 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:51,553 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:51,553 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:51,554 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:52,207 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:52,507 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:52,510 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:52,510 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:52,511 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:53,208 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:53,208 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:53,360 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:53,362 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:53,363 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:53,364 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:54,209 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:54,261 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:54,264 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:54,264 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:54,265 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:54,266 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:55,135 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:55,137 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:55,138 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:55,139 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:55,210 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:55,210 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:56,031 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:56,034 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:56,034 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:56,035 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:56,211 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:56,893 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:56,896 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:56,896 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:56,897 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:57,212 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:57,801 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:57,803 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:57,803 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:57,804 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:58,214 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:58,566 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:58:58,689 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:58,691 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:58,692 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:58,693 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:58:58,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:58:58,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:58:58,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:58:59,215 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:58:59,215 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:58:59,562 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:58:59,564 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:58:59,565 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:58:59,565 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:58:59,566 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:00,216 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:00,635 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:00,637 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:00,638 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:00,639 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:01,217 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:01,217 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:01,528 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:01,530 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:01,531 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:01,532 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:02,218 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:02,413 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:02,415 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:02,416 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:02,417 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:03,220 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:03,220 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:03,305 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:03,309 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:03,309 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:03,310 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:04,221 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:04,263 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:04,295 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:04,295 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:04,296 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:05,152 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:05,155 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:05,156 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:05,156 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:05,157 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:05,222 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:05,222 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:06,050 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:06,052 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:06,053 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:06,054 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:06,223 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:06,945 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:06,950 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:06,950 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:06,951 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:07,224 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:07,909 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:07,911 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:07,912 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:07,913 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:08,225 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:08,804 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:08,807 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:08,807 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:08,808 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:09,227 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:09,227 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:09,813 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:09,816 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:09,816 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:09,817 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:10,228 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:10,688 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:10,690 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:10,691 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:10,691 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:10,692 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:11,229 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:11,229 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:11,639 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:11,641 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:11,641 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:11,643 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:12,230 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:12,528 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:12,530 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:12,530 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:12,532 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:13,231 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:13,232 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:13,482 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:13,485 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:13,485 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:13,486 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:13,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:59:13,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:59:13,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:59:14,233 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:14,377 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:14,379 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:14,379 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:14,380 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:15,234 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:15,234 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:15,254 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:15,257 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:15,257 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:15,258 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:16,143 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:16,146 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:16,147 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:16,147 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:16,148 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:16,235 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:17,155 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:17,157 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:17,158 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:17,159 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:17,236 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:17,236 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:18,038 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:18,041 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:18,041 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:18,043 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:18,237 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:18,914 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:18,917 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:18,917 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:18,918 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:19,238 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:19,783 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:19,786 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:19,786 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:19,787 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:20,240 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:20,682 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:20,685 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:20,685 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:20,687 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:21,241 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:21,241 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:21,572 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:21,576 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:21,576 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:21,576 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:21,578 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:22,242 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:22,456 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:22,458 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:22,459 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:22,461 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:23,243 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:23,243 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:23,355 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:23,360 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:23,361 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:23,362 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:24,244 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:24,260 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:24,262 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:24,263 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:24,264 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:25,146 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:25,148 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:25,149 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:25,151 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:25,246 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:25,246 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:26,153 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:26,156 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:26,156 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:26,157 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:26,247 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:27,058 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:27,060 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:27,061 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:27,061 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:27,062 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:27,248 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:27,248 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:27,956 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:27,958 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:27,959 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:27,960 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:28,567 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:59:28,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:59:28,767 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:59:28,770 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:59:28,837 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:28,940 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:28,941 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:28,942 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:29,250 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:29,250 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:29,731 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:29,734 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:29,734 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:29,735 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:30,253 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:30,625 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:30,628 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:30,628 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:30,629 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:31,255 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:31,255 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:31,518 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:31,520 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:31,521 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:31,522 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:32,256 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:32,414 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:32,416 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:32,417 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:32,417 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:32,418 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:33,257 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:33,257 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:33,285 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:33,287 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:33,287 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:33,289 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:34,259 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:34,286 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:34,288 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:34,289 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:34,290 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:35,173 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:35,176 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:35,176 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:35,178 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:35,260 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:35,260 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:36,061 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:36,063 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:36,064 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:36,067 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:36,261 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:36,945 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:36,950 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:36,950 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:36,952 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:37,262 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:37,812 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:37,814 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:37,815 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:37,815 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:37,817 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:38,263 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:38,692 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:38,695 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:38,696 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:38,697 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:39,265 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:39,265 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:39,594 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:39,598 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:39,598 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:39,599 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:40,266 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:40,466 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:40,469 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:40,469 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:40,473 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:41,267 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:41,267 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:41,357 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:41,359 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:41,360 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:41,361 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:42,234 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:42,237 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:42,237 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:42,239 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:42,268 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:43,239 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:43,246 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:43,250 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:43,251 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:43,253 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:43,269 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:43,270 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:43,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:59:43,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:59:43,770 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:59:44,127 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:44,129 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:44,130 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:44,132 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:44,270 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:45,067 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:45,070 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:45,070 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:45,072 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:45,272 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:45,272 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:45,962 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:45,965 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:45,966 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:45,967 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:46,853 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:46,857 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:46,857 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:46,858 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:47,274 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:47,274 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:47,776 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:47,779 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:47,779 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:47,781 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:48,275 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:48,663 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:48,665 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:48,666 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:48,666 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:48,667 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:49,277 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:49,277 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:49,551 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:49,554 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:49,554 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:49,555 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:50,278 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:50,586 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:50,589 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:50,589 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:50,590 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:51,279 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:51,279 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:51,484 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:51,486 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:51,486 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:51,488 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:52,280 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:52,384 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:52,386 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:52,386 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:52,388 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:53,282 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:53,282 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:53,396 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:53,398 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:53,398 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:53,400 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:54,283 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:54,285 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:54,288 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:54,289 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:54,289 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 20:59:54,290 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:55,176 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:55,178 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:55,179 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:55,180 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:55,284 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:55,284 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:56,066 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 20:59:56,070 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 20:59:56,071 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 20:59:56,072 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 20:59:56,285 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 20:59:57,286 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:58,568 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 20:59:58,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 20:59:58,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 20:59:58,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 20:59:59,288 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 20:59:59,993 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:00,288 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:01,332 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:02,239 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:02,242 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:02,242 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:02,243 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:02,290 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:03,143 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:03,146 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:03,147 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:03,148 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:03,291 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:03,291 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:04,189 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:04,191 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:04,192 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:04,193 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:04,292 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:05,068 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:05,071 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:05,071 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:05,071 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:05,072 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:05,294 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:05,294 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:05,958 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:05,960 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:05,961 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:05,962 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:06,839 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:06,842 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:06,842 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:06,843 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:07,296 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:07,296 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:07,728 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:07,731 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:07,731 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:07,732 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:08,297 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:08,628 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:08,631 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:08,632 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:08,633 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:09,299 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:09,299 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:09,513 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:09,517 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:09,517 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:09,519 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:10,300 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:10,404 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:10,407 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:10,407 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:10,407 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:10,409 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:11,290 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:11,293 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:11,293 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:11,294 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:11,301 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:11,301 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:12,564 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:12,566 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:12,567 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:12,568 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:13,303 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:13,303 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:13,459 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:13,462 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:13,462 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:13,463 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:13,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:00:13,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:00:13,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:00:14,304 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:14,348 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:14,350 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:14,350 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:14,351 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:15,218 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:15,221 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:15,221 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:15,222 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:15,305 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:15,306 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:16,079 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:16,082 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:16,082 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:16,083 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:16,084 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:16,307 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:16,963 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:16,965 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:16,967 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:16,969 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:17,308 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:17,853 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:17,856 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:17,856 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:17,858 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:18,309 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:18,719 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:18,721 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:18,721 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:18,722 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:19,310 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:19,311 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:19,596 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:19,598 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:19,599 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:19,600 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:20,312 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:20,609 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:20,611 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:20,611 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:20,613 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:21,313 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:21,313 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:21,496 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:21,498 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:21,499 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:21,499 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:21,501 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:22,314 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:22,391 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:22,394 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:22,394 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:22,395 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:23,313 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:23,316 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:23,316 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:23,318 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:23,318 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:23,318 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:24,221 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:24,225 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:24,225 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:24,226 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:24,319 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:25,163 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:25,165 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:25,165 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:25,167 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:25,320 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:25,320 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:26,036 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:26,039 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:26,039 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:26,040 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:26,321 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:26,928 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:26,930 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:26,930 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:26,931 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:26,932 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:27,323 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:27,876 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:27,879 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:27,879 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:27,880 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:28,324 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:28,569 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:00:28,726 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:28,729 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:28,729 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:28,731 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:28,767 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:00:28,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:00:28,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:00:29,325 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:29,325 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:29,712 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:29,714 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:29,714 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:29,715 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:30,326 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:30,595 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:30,597 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:30,598 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:30,599 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:31,328 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:31,328 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:31,467 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:31,470 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:31,470 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:31,471 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:32,329 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:32,346 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:32,348 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:32,349 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:32,349 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:32,350 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:33,228 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:33,230 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:33,231 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:33,232 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:33,330 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:33,331 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:34,116 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:34,119 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:34,119 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:34,120 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:34,332 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:35,005 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:35,008 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:35,008 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:35,010 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:35,333 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:35,333 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:35,885 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:35,887 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:35,888 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:35,889 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:36,776 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:36,779 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:36,779 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:36,780 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:37,336 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:37,336 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:37,781 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:37,785 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:37,787 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:37,788 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:37,789 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:38,337 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:38,679 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:38,682 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:38,682 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:38,684 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:39,338 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:39,339 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:39,578 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:39,580 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:39,581 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:39,582 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:40,340 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:40,457 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:40,459 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:40,459 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:40,461 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:41,341 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:41,341 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:41,347 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:41,350 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:41,350 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:41,351 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:42,248 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:42,251 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:42,252 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:42,253 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:42,342 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:43,126 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:43,128 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:43,129 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:43,129 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:43,131 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:43,344 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:43,344 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:43,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:00:43,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:00:43,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:00:44,015 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:44,029 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:44,029 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:44,031 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:44,345 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:44,894 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:44,896 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:44,896 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:44,897 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:45,346 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:45,778 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:45,806 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:45,806 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:45,807 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:46,347 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:46,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:46,777 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:46,777 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:46,779 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:47,349 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:47,349 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:47,668 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:47,671 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:47,671 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:47,672 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:48,350 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:48,572 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:48,575 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:48,576 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:48,576 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:48,577 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:49,351 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:49,351 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:49,455 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:49,457 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:49,458 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:49,459 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:50,334 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:50,338 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:50,339 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:50,340 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:50,352 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:51,220 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:51,223 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:51,224 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:51,225 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:51,353 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:51,354 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:52,094 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:52,097 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:52,098 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:52,099 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:52,355 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:52,984 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:52,986 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:52,987 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:52,988 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:53,356 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:53,853 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:53,856 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:53,856 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:53,856 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:53,857 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:54,357 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:54,857 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:54,863 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:54,864 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:54,865 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:55,359 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:55,359 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:55,741 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:55,745 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:55,745 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:55,746 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:56,360 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:56,619 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:56,622 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:56,622 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:56,623 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:57,362 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:57,362 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:57,506 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:57,509 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:57,509 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:57,510 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:58,363 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:00:58,393 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:58,397 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:58,397 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:58,398 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:58,569 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:00:58,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:00:58,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:00:58,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:00:58,966 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:00:59,287 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:00:59,288 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:00:59,289 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:00:59,290 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:00:59,365 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:00:59,365 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:00,190 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:00,193 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:00,193 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:00,194 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:00,367 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:01,085 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:01,087 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:01,088 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:01,089 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:01,368 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:01,369 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:01,984 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:01,987 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:01,987 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:01,988 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:02,984 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:02,986 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:02,987 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:02,988 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:03,371 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:03,371 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:03,885 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:03,888 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:03,888 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:03,889 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:04,373 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:04,776 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:04,778 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:04,779 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:04,779 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:04,780 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:05,374 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:05,375 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:05,682 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:05,685 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:05,685 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:05,687 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:06,376 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:06,578 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:06,580 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:06,580 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:06,581 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:07,377 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:07,377 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:07,465 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:07,468 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:07,468 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:07,470 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:08,341 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:08,343 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:08,343 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:08,344 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:08,378 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:09,224 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:09,226 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:09,227 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:09,228 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:09,379 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:09,379 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:10,118 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:10,120 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:10,121 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:10,121 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:10,122 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:10,380 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:11,115 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:11,118 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:11,119 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:11,120 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:11,382 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:11,382 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:12,005 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:12,007 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:12,008 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:12,009 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:12,383 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:12,886 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:12,889 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:12,890 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:12,891 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:13,384 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:13,754 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:13,757 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:13,757 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:13,758 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:13,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:01:13,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:01:13,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:01:14,385 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:14,651 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:14,652 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:14,653 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:14,654 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:15,387 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:15,387 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:15,547 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:15,550 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:15,550 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:15,550 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:15,552 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:16,388 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:16,427 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:16,430 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:16,431 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:16,432 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:17,309 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:17,311 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:17,312 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:17,313 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:17,389 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:17,389 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:18,254 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:18,257 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:18,257 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:18,258 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:18,390 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:19,126 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:19,128 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:19,129 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:19,130 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:19,391 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:19,392 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:20,115 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:20,118 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:20,118 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:20,119 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:20,393 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:20,987 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:20,991 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:20,991 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:20,992 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:20,993 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:21,394 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:21,932 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:21,934 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:21,934 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:21,936 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:22,395 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:22,811 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:22,814 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:22,814 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:22,815 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:23,396 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:23,397 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:23,708 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:23,711 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:23,711 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:23,712 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:24,398 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:24,586 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:24,588 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:24,589 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:24,590 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:25,399 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:25,399 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:25,493 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:25,495 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:25,496 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:25,497 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:26,390 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:26,396 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:26,396 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:26,396 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:26,398 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:26,400 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:27,299 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:27,301 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:27,302 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:27,303 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:27,401 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:27,402 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:28,305 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:28,308 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:28,309 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:28,310 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:28,402 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:28,571 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:01:28,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:01:28,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:01:28,772 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:01:29,303 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:29,305 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:29,306 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:29,307 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:29,404 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:29,404 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:30,245 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:30,247 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:30,248 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:30,249 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:30,405 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:31,145 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:31,149 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:31,149 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:31,150 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:31,406 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:31,406 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:32,052 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:32,055 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:32,055 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:32,055 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:32,056 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:32,407 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:33,408 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:35,410 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:36,410 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:37,303 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:37,305 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:38,207 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:38,210 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:38,210 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:38,212 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:38,412 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:39,108 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:39,111 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:39,111 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:39,112 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:39,413 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:39,414 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:40,000 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:40,004 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:40,004 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:40,005 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:40,415 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:41,034 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:41,037 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:41,037 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:41,038 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:41,416 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:41,416 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:41,911 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:41,913 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:41,913 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:41,915 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:42,783 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:42,786 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:42,786 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:42,787 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:42,788 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:43,418 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:43,419 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:43,677 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:43,680 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:43,680 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:43,681 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:43,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:01:43,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:01:43,773 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:01:44,420 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:44,583 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:44,584 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:44,585 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:44,586 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:45,421 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:45,421 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:45,476 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:45,479 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:45,480 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:45,481 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:46,376 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:46,379 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:46,379 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:46,381 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:46,422 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:47,271 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:47,274 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:47,274 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:47,275 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:47,423 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:47,424 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:48,276 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:48,352 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:48,355 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:48,355 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:48,356 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:48,425 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:49,335 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:49,337 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:49,338 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:49,339 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:49,426 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:49,426 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:50,331 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:50,334 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:50,334 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:50,335 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:50,427 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:51,225 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:51,228 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:51,228 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:51,229 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:51,429 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:51,429 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:52,107 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:52,110 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:52,111 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:52,112 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:52,430 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:53,003 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:53,006 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:53,006 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:53,007 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:53,431 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:53,431 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:53,913 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:53,915 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:53,916 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:53,916 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:53,917 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:54,788 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:54,789 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:54,791 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:54,792 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:55,434 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:55,434 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:55,677 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:55,680 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:55,680 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:55,681 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:56,435 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:56,586 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:56,588 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:56,589 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:56,590 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:57,436 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:57,436 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:57,602 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:57,604 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:57,605 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:57,608 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:58,437 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:01:58,488 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:58,491 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:58,491 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:58,492 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:58,571 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:01:58,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:01:58,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:01:58,773 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:01:58,973 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:01:59,398 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:01:59,400 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:01:59,400 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:01:59,401 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:01:59,438 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:01:59,439 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:00,300 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:00,303 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:00,303 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:00,305 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:00,440 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:01,205 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:01,207 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:01,208 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:01,209 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:01,441 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:01,441 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:02,117 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:02,119 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:02,120 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:02,121 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:02,442 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:03,010 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:03,012 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:03,013 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:03,014 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:03,443 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:03,444 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:03,874 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:03,876 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:03,876 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:03,878 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:04,739 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:04,742 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:04,743 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:04,743 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:04,744 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:05,446 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:05,446 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:05,650 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:05,652 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:05,653 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:05,654 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:06,447 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:06,560 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:06,563 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:06,563 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:06,564 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:07,448 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:07,449 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:07,574 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:07,577 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:07,577 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:07,578 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:08,446 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:08,449 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:08,449 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:08,450 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:08,450 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:09,352 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:09,354 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:09,355 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:09,357 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:09,452 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:09,452 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:10,267 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:10,269 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:10,270 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:10,270 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:10,271 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:10,453 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:11,146 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:11,148 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:11,148 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:11,150 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:11,454 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:11,454 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:12,039 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:12,041 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:12,041 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:12,043 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:12,455 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:12,918 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:12,921 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:12,922 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:12,923 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:13,456 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:13,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:02:13,768 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:02:13,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:02:13,795 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:13,996 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:13,996 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:13,997 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:14,457 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:14,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:14,778 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:14,778 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:14,779 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:15,459 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:15,459 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:15,662 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:15,664 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:15,665 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:15,665 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:15,666 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:16,460 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:16,538 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:16,540 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:16,541 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:16,542 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:17,427 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:17,454 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:17,454 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:17,455 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:17,461 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:17,461 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:18,324 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:18,327 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:18,327 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:18,328 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:18,462 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:19,229 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:19,231 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:19,232 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:19,233 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:19,463 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:19,463 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:20,108 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:20,111 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:20,111 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:20,112 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:20,464 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:20,994 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:20,996 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:20,996 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:20,997 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:20,997 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:21,465 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:21,861 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:21,863 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:21,863 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:21,864 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:22,466 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:22,742 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:22,744 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:22,745 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:22,746 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:23,468 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:23,468 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:23,815 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:23,817 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:23,818 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:23,820 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:24,469 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:24,686 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:24,689 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:24,690 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:24,691 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:25,470 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:25,470 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:25,573 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:25,576 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:25,576 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:25,577 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:26,440 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:26,442 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:26,443 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:26,443 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:26,444 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:26,471 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:27,324 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:27,327 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:27,327 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:27,328 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:27,472 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:27,472 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:28,213 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:28,215 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:28,216 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:28,217 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:28,473 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:28,572 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:02:28,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:02:28,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:02:28,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:02:29,079 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:29,082 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:29,083 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:29,084 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:29,474 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:29,474 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:29,952 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:29,954 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:29,954 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:29,955 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:30,818 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:30,821 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:30,821 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:30,822 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:31,476 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:31,477 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:31,823 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:31,925 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:31,928 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:31,928 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:31,929 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:32,477 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:32,803 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:32,805 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:32,806 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:32,807 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:33,479 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:33,479 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:33,680 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:33,683 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:33,683 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:33,685 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:34,480 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:34,577 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:34,579 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:34,580 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:34,581 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:35,479 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:35,482 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:35,482 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:35,483 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:35,483 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:35,483 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:36,366 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:36,369 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:36,370 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:36,371 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:36,484 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:37,268 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:37,270 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:37,271 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:37,271 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:37,272 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:37,485 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:37,485 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:38,153 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:38,155 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:38,156 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:38,157 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:38,486 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:39,051 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:39,054 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:39,054 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:39,055 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:39,488 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:39,488 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:39,927 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:39,929 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:39,929 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:39,931 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:40,935 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:40,938 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:40,938 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:40,940 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:41,490 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:41,490 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:41,816 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:41,818 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:41,819 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:41,820 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:42,491 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:42,698 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:42,700 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:42,701 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:42,701 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:42,702 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:43,493 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:43,493 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:43,596 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:43,598 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:43,598 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:43,600 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:43,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:02:43,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:02:43,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:02:44,467 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:44,469 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:44,470 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:44,471 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:44,494 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:45,350 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:45,353 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:45,354 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:45,355 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:45,495 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:45,495 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:46,346 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:46,349 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:46,349 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:46,351 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:46,496 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:47,311 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:47,314 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:47,314 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:47,315 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:47,498 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:47,498 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:48,308 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:48,311 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:48,312 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:48,312 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:48,313 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:48,499 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:49,172 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:49,175 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:49,175 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:49,176 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:49,500 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:49,500 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:50,048 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:50,050 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:50,050 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:50,052 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:50,501 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:50,947 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:50,950 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:50,950 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:50,951 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:51,502 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:51,822 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:51,825 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:51,825 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:51,826 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:52,504 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:52,718 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:52,720 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:52,721 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:52,722 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:53,505 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:53,505 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:53,595 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:53,597 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:53,597 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:53,597 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:53,598 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:54,460 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:54,464 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:54,464 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:54,465 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:54,506 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:55,334 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:55,336 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:55,337 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:55,338 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:55,507 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:55,507 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:56,206 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:56,209 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:56,209 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:56,211 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:56,508 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:57,198 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:57,202 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:57,202 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:57,203 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:57,509 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:57,510 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:58,093 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:58,096 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:58,096 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:58,097 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:58,511 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:02:58,573 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:02:58,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:02:58,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:02:58,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:02:58,968 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:58,985 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:58,985 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:58,985 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:02:58,987 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:02:59,512 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:02:59,853 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:02:59,855 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:02:59,855 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:02:59,856 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:00,513 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:00,744 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:00,747 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:00,747 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:00,749 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:01,515 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:01,515 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:01,683 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:01,686 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:01,687 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:01,688 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:02,516 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:02,622 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:02,625 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:02,625 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:02,626 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:03,517 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:03,517 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:03,519 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:03,521 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:03,522 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:03,523 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:04,504 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:04,507 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:04,507 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:04,507 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:04,509 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:04,518 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:05,519 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:05,561 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:05,564 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:05,564 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:05,565 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:06,447 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:06,449 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:06,449 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:06,451 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:06,520 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:07,339 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:07,342 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:07,342 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:07,343 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:07,521 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:07,522 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:08,239 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:08,241 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:08,241 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:08,243 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:08,523 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:09,523 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:10,018 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:11,525 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:12,526 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:13,623 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:13,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:03:13,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:03:13,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:03:14,536 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:14,538 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:14,538 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:14,539 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:15,431 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:15,434 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:15,434 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:15,434 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:15,435 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:15,528 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:15,528 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:16,361 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:16,364 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:16,364 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:16,365 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:16,529 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:17,253 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:17,255 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:17,256 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:17,257 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:17,530 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:17,531 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:18,134 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:18,137 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:18,137 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:18,138 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:18,532 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:19,174 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:19,176 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:19,177 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:19,178 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:19,533 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:19,534 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:20,072 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:20,075 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:20,075 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:20,076 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:20,534 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:20,953 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:20,955 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:20,956 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:20,956 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:20,957 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:21,536 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:21,870 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:21,874 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:21,874 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:21,875 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:22,537 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:22,752 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:22,754 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:22,754 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:22,756 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:23,538 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:23,539 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:23,777 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:23,780 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:23,781 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:23,782 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:24,540 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:24,678 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:24,680 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:24,681 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:24,682 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:25,541 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:25,541 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:25,574 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:25,577 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:25,577 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:25,578 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:26,433 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:26,435 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:26,436 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:26,436 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:26,438 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:26,542 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:27,296 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:27,299 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:27,299 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:27,301 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:27,543 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:27,543 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:28,330 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:28,332 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:28,333 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:28,334 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:28,544 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:28,574 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:03:28,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:03:28,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:03:28,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:03:29,228 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:29,229 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:29,230 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:29,231 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:29,546 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:29,546 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:30,096 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:30,099 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:30,099 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:30,101 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:30,547 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:30,969 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:30,971 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:30,972 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:30,974 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:31,548 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:31,868 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:31,871 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:31,871 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:31,871 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:31,873 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:32,549 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:32,758 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:32,760 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:32,761 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:32,762 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:33,550 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:33,551 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:33,660 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:33,663 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:33,663 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:33,664 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:34,552 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:34,569 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:34,571 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:34,571 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:34,572 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:35,553 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:35,553 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:35,598 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:35,600 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:35,601 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:35,602 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:36,496 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:36,498 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:36,499 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:36,500 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:36,554 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:37,374 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:37,377 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:37,377 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:37,378 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:37,379 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:37,555 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:37,555 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:38,340 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:38,342 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:38,342 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:38,344 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:38,556 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:39,205 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:39,207 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:39,207 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:39,209 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:39,558 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:39,558 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:40,051 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:40,054 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:40,054 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:40,056 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:40,559 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:40,943 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:40,946 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:40,946 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:40,947 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:41,560 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:41,824 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:41,827 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:41,827 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:41,829 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:42,561 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:42,701 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:42,704 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:42,704 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:42,704 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:42,705 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:43,562 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:43,563 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:43,579 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:43,582 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:43,582 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:43,583 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:43,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:03:43,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:03:43,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:03:44,561 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:44,563 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:44,563 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:44,564 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:44,565 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:45,514 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:45,517 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:45,517 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:45,519 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:45,566 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:45,566 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:46,382 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:46,385 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:46,385 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:46,390 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:46,567 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:47,262 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:47,265 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:47,265 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:47,267 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:47,568 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:47,568 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:48,152 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:48,155 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:48,155 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:48,156 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:48,158 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:48,569 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:49,043 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:49,046 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:49,046 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:49,048 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:49,570 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:49,571 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:49,940 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:49,943 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:49,943 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:49,944 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:50,890 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:50,893 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:50,893 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:50,895 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:51,573 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:51,573 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:51,768 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:51,771 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:51,771 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:51,772 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:52,574 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:52,763 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:52,765 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:52,765 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:52,767 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:53,575 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:53,576 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:53,669 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:53,672 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:53,672 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:53,672 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:53,673 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:54,555 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:54,559 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:54,559 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:54,560 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:54,577 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:55,439 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:55,465 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:55,465 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:55,467 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:55,578 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:55,578 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:56,315 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:56,317 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:56,318 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:56,319 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:56,579 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:57,279 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:57,282 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:57,282 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:57,283 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:57,580 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:57,580 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:58,151 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:58,153 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:58,154 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:58,155 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:58,575 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:03:58,581 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:58,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:03:58,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:03:58,774 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:03:59,000 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:03:59,036 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:59,037 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:59,037 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:59,039 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:03:59,582 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:03:59,583 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:03:59,941 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:03:59,945 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:03:59,945 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:03:59,946 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:00,879 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:00,882 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:00,882 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:00,883 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:01,585 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:01,585 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:01,868 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:01,871 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:01,872 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:01,873 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:02,586 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:02,760 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:02,763 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:02,763 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:02,764 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:03,587 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:03,588 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:03,703 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:03,706 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:03,707 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:03,708 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:04,589 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:04,684 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:04,687 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:04,687 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:04,688 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:04,688 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:05,547 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:05,550 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:05,550 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:05,551 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:05,590 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:05,590 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:06,497 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:06,499 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:06,500 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:06,501 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:06,591 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:07,431 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:07,434 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:07,434 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:07,435 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:07,592 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:07,592 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:08,313 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:08,315 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:08,316 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:08,317 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:08,593 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:09,320 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:09,322 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:09,323 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:09,324 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:09,594 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:09,595 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:10,217 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:10,219 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:10,220 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:10,221 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:10,222 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:10,596 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:11,110 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:11,113 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:11,113 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:11,114 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:11,597 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:11,597 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:12,015 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:12,017 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:12,017 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:12,019 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:12,598 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:12,910 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:12,912 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:12,913 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:12,914 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:13,599 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:13,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:04:13,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:04:13,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:04:13,803 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:13,996 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:13,997 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:13,998 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:14,600 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:14,687 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:14,690 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:14,690 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:14,691 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:15,585 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:15,588 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:15,588 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:15,589 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:15,590 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:15,602 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:15,602 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:16,465 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:16,467 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:16,468 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:16,469 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:16,603 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:17,362 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:17,365 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:17,365 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:17,366 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:17,604 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:17,604 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:18,367 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:18,370 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:18,370 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:18,371 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:18,605 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:19,250 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:19,252 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:19,253 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:19,256 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:19,606 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:19,606 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:20,127 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:20,130 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:20,130 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:20,131 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:20,607 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:21,006 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:21,009 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:21,010 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:21,010 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:21,011 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:21,609 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:21,609 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:21,902 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:21,905 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:21,905 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:21,907 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:22,783 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:22,788 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:22,788 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:22,789 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:23,611 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:23,611 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:23,673 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:23,676 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:23,677 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:23,678 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:24,556 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:24,558 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:24,559 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:24,562 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:24,612 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:25,417 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:25,420 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:25,421 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:25,422 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:25,614 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:25,614 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:26,422 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:26,425 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:26,428 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:26,428 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:26,430 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:26,615 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:27,319 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:27,324 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:27,324 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:27,325 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:27,616 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:27,616 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:28,191 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:28,194 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:28,195 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:28,196 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:28,576 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:04:28,617 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:28,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:04:28,769 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:04:28,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:04:29,073 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:29,075 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:29,075 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:29,076 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:29,618 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:29,619 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:29,956 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:29,958 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:29,959 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:29,960 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:30,820 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:30,823 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:30,823 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:30,824 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:31,621 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:31,621 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:31,702 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:31,705 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:31,705 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:31,705 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:31,707 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:32,577 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:32,580 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:32,580 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:32,581 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:32,622 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:33,481 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:33,484 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:33,484 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:33,486 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:33,623 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:33,623 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:34,350 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:34,352 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:34,353 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:34,354 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:34,625 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:35,341 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:35,345 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:35,345 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:35,346 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:35,626 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:35,626 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:36,217 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:36,219 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:36,220 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:36,221 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:36,628 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:37,126 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:37,128 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:37,128 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:37,129 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:37,130 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:37,629 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:37,630 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:38,050 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:38,053 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:38,054 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:38,055 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:38,631 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:38,962 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:38,965 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:38,965 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:38,966 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:39,632 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:39,846 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:39,849 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:39,849 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:39,850 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:40,634 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:40,840 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:40,843 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:40,844 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:40,845 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:41,635 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:41,635 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:41,728 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:41,730 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:41,730 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:41,731 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:42,636 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:42,732 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:42,780 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:42,783 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:42,783 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:42,784 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:43,638 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:43,638 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:43,673 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:43,677 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:43,677 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:43,678 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:43,769 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: stop_status +2024-08-31 21:04:43,770 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: stop_status +2024-08-31 21:04:43,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:04:44,576 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:44,578 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:44,578 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:44,579 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:44,639 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:45,639 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:47,641 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:48,160 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:48,641 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:49,775 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: partial_history +2024-08-31 21:04:51,643 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:52,644 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:53,391 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:54,404 DEBUG SenderThread:15237 [sender.py:send():382] send: exit +2024-08-31 21:04:54,404 INFO SenderThread:15237 [sender.py:send_exit():589] handling exit code: 0 +2024-08-31 21:04:54,404 INFO SenderThread:15237 [sender.py:send_exit():591] handling runtime: 985 +2024-08-31 21:04:54,405 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:54,406 INFO SenderThread:15237 [sender.py:send_exit():597] send defer +2024-08-31 21:04:54,406 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,406 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-31 21:04:54,406 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,406 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-31 21:04:54,406 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 1 +2024-08-31 21:04:54,406 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,406 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-31 21:04:54,407 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,407 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-31 21:04:54,407 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 2 +2024-08-31 21:04:54,407 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,407 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-31 21:04:54,407 INFO HandlerThread:15237 [system_monitor.py:finish():203] Stopping system monitor +2024-08-31 21:04:54,407 INFO HandlerThread:15237 [interfaces.py:finish():202] Joined cpu monitor +2024-08-31 21:04:54,407 DEBUG SystemMonitor:15237 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-31 21:04:54,407 INFO HandlerThread:15237 [interfaces.py:finish():202] Joined disk monitor +2024-08-31 21:04:54,408 DEBUG SystemMonitor:15237 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-31 21:04:54,442 INFO HandlerThread:15237 [interfaces.py:finish():202] Joined gpu monitor +2024-08-31 21:04:54,442 INFO HandlerThread:15237 [interfaces.py:finish():202] Joined memory monitor +2024-08-31 21:04:54,442 INFO HandlerThread:15237 [interfaces.py:finish():202] Joined network monitor +2024-08-31 21:04:54,442 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,442 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-31 21:04:54,443 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 3 +2024-08-31 21:04:54,443 DEBUG SenderThread:15237 [sender.py:send():382] send: stats +2024-08-31 21:04:54,443 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,443 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-31 21:04:54,444 DEBUG SenderThread:15237 [sender.py:send():382] send: history +2024-08-31 21:04:54,445 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: summary_record +2024-08-31 21:04:54,446 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:54,446 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,446 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-31 21:04:54,446 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 4 +2024-08-31 21:04:54,446 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,446 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-31 21:04:54,446 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,446 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-31 21:04:54,446 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 5 +2024-08-31 21:04:54,446 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,446 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-31 21:04:54,447 DEBUG SenderThread:15237 [sender.py:send():382] send: summary +2024-08-31 21:04:54,448 INFO SenderThread:15237 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-31 21:04:54,448 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,448 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-31 21:04:54,448 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 6 +2024-08-31 21:04:54,448 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,449 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-31 21:04:54,449 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,449 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-31 21:04:54,449 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 7 +2024-08-31 21:04:54,449 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: status_report +2024-08-31 21:04:54,449 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:54,449 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-31 21:04:54,449 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:54,449 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-31 21:04:54,646 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:55,404 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 21:04:55,647 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:56,553 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 8 +2024-08-31 21:04:56,553 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 21:04:56,554 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:56,554 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-31 21:04:56,554 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:56,554 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-31 21:04:56,554 INFO SenderThread:15237 [job_builder.py:build():296] Attempting to build job artifact +2024-08-31 21:04:56,555 INFO SenderThread:15237 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-31 21:04:56,571 INFO SenderThread:15237 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-31 21:04:56,581 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 9 +2024-08-31 21:04:56,581 DEBUG SenderThread:15237 [sender.py:send():382] send: artifact +2024-08-31 21:04:56,581 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:56,582 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-31 21:04:56,647 INFO Thread-12 :15237 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:57,405 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 21:04:57,441 INFO SenderThread:15237 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE3NDAzNTQ3MA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjgwMzg3NA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE3NDAzNTQ3MA==', 'versionIndex': 3}}} +2024-08-31 21:04:57,441 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:57,441 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-31 21:04:57,441 INFO SenderThread:15237 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-31 21:04:57,648 INFO SenderThread:15237 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240831_204827-vaha0m5b/files +2024-08-31 21:04:57,649 INFO SenderThread:15237 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_204827-vaha0m5b/files/requirements.txt requirements.txt +2024-08-31 21:04:57,649 INFO SenderThread:15237 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_204827-vaha0m5b/files/config.yaml config.yaml +2024-08-31 21:04:57,651 INFO SenderThread:15237 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-metadata.json wandb-metadata.json +2024-08-31 21:04:57,651 INFO SenderThread:15237 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json wandb-summary.json +2024-08-31 21:04:57,652 INFO SenderThread:15237 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240831_204827-vaha0m5b/files/output.log output.log +2024-08-31 21:04:57,654 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 10 +2024-08-31 21:04:57,654 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 21:04:57,655 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:57,656 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-31 21:04:57,657 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:57,657 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-31 21:04:57,657 INFO SenderThread:15237 [file_pusher.py:finish():172] shutting down file pusher +2024-08-31 21:04:58,110 INFO wandb-upload_1:15237 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_204827-vaha0m5b/files/config.yaml +2024-08-31 21:04:58,114 INFO wandb-upload_2:15237 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_204827-vaha0m5b/files/wandb-summary.json +2024-08-31 21:04:58,121 INFO wandb-upload_0:15237 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_204827-vaha0m5b/files/requirements.txt +2024-08-31 21:04:58,405 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 21:04:58,406 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 21:04:58,560 INFO wandb-upload_3:15237 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240831_204827-vaha0m5b/files/output.log +2024-08-31 21:04:58,760 INFO Thread-11 (_thread_body):15237 [sender.py:transition_state():617] send defer: 11 +2024-08-31 21:04:58,760 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:58,760 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-31 21:04:58,761 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:58,761 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-31 21:04:58,761 INFO SenderThread:15237 [file_pusher.py:join():178] waiting for file pusher +2024-08-31 21:04:58,761 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 12 +2024-08-31 21:04:58,761 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:58,761 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-31 21:04:58,761 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:58,761 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-31 21:04:58,761 INFO SenderThread:15237 [file_stream.py:finish():595] file stream finish called +2024-08-31 21:04:59,357 INFO SenderThread:15237 [file_stream.py:finish():599] file stream finish is done +2024-08-31 21:04:59,357 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 13 +2024-08-31 21:04:59,357 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:59,357 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-31 21:04:59,357 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:59,357 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-31 21:04:59,357 INFO SenderThread:15237 [sender.py:transition_state():617] send defer: 14 +2024-08-31 21:04:59,358 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: defer +2024-08-31 21:04:59,358 DEBUG SenderThread:15237 [sender.py:send():382] send: final +2024-08-31 21:04:59,358 INFO HandlerThread:15237 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-31 21:04:59,358 DEBUG SenderThread:15237 [sender.py:send():382] send: footer +2024-08-31 21:04:59,358 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: defer +2024-08-31 21:04:59,358 INFO SenderThread:15237 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-31 21:04:59,359 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 21:04:59,359 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-31 21:04:59,359 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: server_info +2024-08-31 21:04:59,359 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: get_summary +2024-08-31 21:04:59,360 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-31 21:04:59,366 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-31 21:04:59,366 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 21:04:59,366 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: poll_exit +2024-08-31 21:04:59,367 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: server_info +2024-08-31 21:04:59,368 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: job_info +2024-08-31 21:04:59,555 DEBUG SenderThread:15237 [sender.py:send_request():409] send_request: job_info +2024-08-31 21:04:59,555 INFO MainThread:15237 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-31 21:04:59,557 INFO MainThread:15237 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-31 21:04:59,557 INFO MainThread:15237 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-31 21:04:59,558 DEBUG HandlerThread:15237 [handler.py:handle_request():146] handle_request: shutdown +2024-08-31 21:04:59,558 INFO HandlerThread:15237 [handler.py:finish():869] shutting down handler +2024-08-31 21:05:00,368 INFO WriterThread:15237 [datastore.py:close():296] close: /project/wandb/run-20240831_204827-vaha0m5b/run-vaha0m5b.wandb +2024-08-31 21:05:00,555 INFO SenderThread:15237 [sender.py:finish():1572] shutting down sender +2024-08-31 21:05:00,556 INFO SenderThread:15237 [file_pusher.py:finish():172] shutting down file pusher +2024-08-31 21:05:00,556 INFO SenderThread:15237 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240831_204827-vaha0m5b/logs/debug.log b/wandb/run-20240831_204827-vaha0m5b/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0e5466cd983d80299e36ad9ae69684220eae8061 --- /dev/null +++ b/wandb/run-20240831_204827-vaha0m5b/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-31 20:48:27,850 INFO MainThread:15165 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_setup.py:_flush():76] Configure stats pid to 15165 +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240831_204827-vaha0m5b/logs/debug.log +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240831_204827-vaha0m5b/logs/debug-internal.log +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_init.py:init():566] calling init triggers +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'SHARD_GRAD_OP', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': None, 'valid_data_path': None, 'test_data_path': None, 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara_train_2024-08-31-20:48:03', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 100, 'save_interval': 100, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 1000, 'lr_warmup_iters': 25, 'min_lr': 1e-06, 'train_iters': 1000, 'train_samples': None, 'global_batch_size': 16, 'micro_batch_size': 4, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.99, 'adam_eps': 1e-06, 'hf_transformer_model_dir': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'instruction_train_data_path': '/work/datasets/bin/ichikara/train/data', 'instruction_valid_data_path': '/work/datasets/bin/ichikara/valid/data', 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': True, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B-3.5e-5-last5-ichikara', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': 'source', 'target_key': 'target', 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': True, 'remove_padding_masking': True, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 4} +2024-08-31 20:48:27,851 INFO MainThread:15165 [wandb_init.py:init():616] starting backend +2024-08-31 20:48:27,852 INFO MainThread:15165 [wandb_init.py:init():620] setting up manager +2024-08-31 20:48:27,857 INFO MainThread:15165 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-31 20:48:27,857 INFO MainThread:15165 [wandb_init.py:init():628] backend started and connected +2024-08-31 20:48:27,862 INFO MainThread:15165 [wandb_init.py:init():720] updated telemetry +2024-08-31 20:48:28,038 INFO MainThread:15165 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-31 20:48:28,465 INFO MainThread:15165 [wandb_run.py:_on_init():2262] communicating current version +2024-08-31 20:48:28,490 INFO MainThread:15165 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.8 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-31 20:48:28,490 INFO MainThread:15165 [wandb_init.py:init():804] starting run threads in backend +2024-08-31 20:48:28,591 INFO MainThread:15165 [wandb_run.py:_console_start():2241] atexit reg +2024-08-31 20:48:28,591 INFO MainThread:15165 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-31 20:48:28,591 INFO MainThread:15165 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-31 20:48:28,591 INFO MainThread:15165 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-31 20:48:28,592 INFO MainThread:15165 [wandb_init.py:init():847] run started, returning control to user process +2024-08-31 20:48:34,071 INFO MainThread:15165 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-31 20:48:34,071 INFO MainThread:15165 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-31 21:05:00,558 WARNING MsgRouterThr:15165 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241031_225437-1g18vgup/files/config.yaml b/wandb/run-20241031_225437-1g18vgup/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..231a93522b39628bbe57853f64f6356fb5ae6252 --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/files/config.yaml @@ -0,0 +1,340 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '183190553' + - /project/trans/datasets/train/documents_text_document +valid_data_path: + desc: null + value: + - '61085876' + - /project/trans/datasets/valid/documents_text_document +test_data_path: + desc: null + value: + - '61085876' + - /project/trans/datasets/valid/documents_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-22:54:20 +wandb_project: + desc: null + value: trans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs +save: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs +base_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 20 +save_interval: + desc: null + value: 50 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 0.00015 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 698 +lr_warmup_iters: + desc: null + value: 70 +min_lr: + desc: null + value: 1.5e-05 +train_iters: + desc: null + value: 698 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 256 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 3 +hf_repo_id: + desc: null + value: koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 99584 +gradient_accumulation_steps: + desc: null + value: 256 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1730382877.322195 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: LlamaForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 3072 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 24 +num_hidden_layers: + desc: null + value: 28 diff --git a/wandb/run-20241031_225437-1g18vgup/files/output.log b/wandb/run-20241031_225437-1g18vgup/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..5004767a78c3a0338e514646593fdb40fcbbcce4 --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/files/output.log @@ -0,0 +1,2400 @@ +Created Hugging Face repository with ID koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` + +Loading checkpoint shards: 50%|█████ | 1/2 [01:22<01:22, 82.50s/it] +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs, skipping model loading +--> Model /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b + +Loading checkpoint shards: 100%|██████████| 2/2 [02:02<00:00, 61.06s/it] +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 178688 + validation: 89600 + test: 2560 +> building train, validation, and test datasets for GPT ... +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs, skipping optimizer loading +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(99584, 3072) + (layers): ModuleList( + (0-27): 28 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=3072, out_features=3072, bias=False) + (k_proj): Linear(in_features=3072, out_features=3072, bias=False) + (v_proj): Linear(in_features=3072, out_features=3072, bias=False) + (o_proj): Linear(in_features=3072, out_features=3072, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=3072, out_features=8192, bias=False) + (up_proj): Linear(in_features=3072, out_features=8192, bias=False) + (down_proj): Linear(in_features=8192, out_features=3072, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=3072, out_features=99584, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 24, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 99584 +} +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 89.70907110365386, Tokens per sec: 4092.7739856871003, Loss: 3.0479860305786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 91.6738975807786, Tokens per sec: 4182.414761062801, Loss: 2.86543869972229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 92.14845568693477, Tokens per sec: 4204.065404054398, Loss: 2.731929063796997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 91.63579805017402, Tokens per sec: 4180.6765559314035, Loss: 2.69535493850708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 91.58376672405888, Tokens per sec: 4178.302744059927, Loss: 2.649918794631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 91.38343655927157, Tokens per sec: 4169.163132235806, Loss: 2.5929694175720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 92.59798318252058, Tokens per sec: 4224.574081907708, Loss: 2.6081435680389404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 91.54590519357431, Tokens per sec: 4176.575397147079, Loss: 2.587522029876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 91.22129535523547, Tokens per sec: 4161.765805591769, Loss: 2.564058303833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 92.1564792931023, Tokens per sec: 4204.431462984557, Loss: 2.544379711151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 91.64988731957042, Tokens per sec: 4181.319347062261, Loss: 2.5465874671936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 91.93731293963923, Tokens per sec: 4194.432492546493, Loss: 2.5238845348358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 91.93834076922239, Tokens per sec: 4194.479384952445, Loss: 2.5077037811279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 91.54963708769506, Tokens per sec: 4176.745656397194, Loss: 2.500976800918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 91.87377192940055, Tokens per sec: 4191.533577302698, Loss: 2.507295846939087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 91.65389445085015, Tokens per sec: 4181.502163386822, Loss: 2.4769535064697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 91.33997968986827, Tokens per sec: 4167.180510607859, Loss: 2.4917523860931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 91.17232913951582, Tokens per sec: 4159.531832467208, Loss: 2.49306058883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 91.72356062684104, Tokens per sec: 4184.680525499705, Loss: 2.4546632766723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 91.89011737680343, Tokens per sec: 4192.27930146517, Loss: 2.458678960800171 +------------------------------------------------------------------ + eval ppl=10.97310733795166, eval loss=2.3954474925994873 +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 91.37237973934451, Tokens per sec: 4168.658689770789, Loss: 2.458890438079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 92.21026103392248, Tokens per sec: 4206.885133578019, Loss: 2.445844888687134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 92.03233175136903, Tokens per sec: 4198.767511469489, Loss: 2.446026563644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 92.40099818681357, Tokens per sec: 4215.587085876184, Loss: 2.434105396270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 92.93416605569605, Tokens per sec: 4239.911667068705, Loss: 2.4301018714904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 92.11334082166, Tokens per sec: 4202.463367545275, Loss: 2.42971134185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 92.06263073332367, Tokens per sec: 4200.149834166749, Loss: 2.413586139678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 92.01586190708831, Tokens per sec: 4198.016111980104, Loss: 2.40895676612854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 91.66809393088798, Tokens per sec: 4182.149982629559, Loss: 2.41538405418396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 92.03626338546626, Tokens per sec: 4198.946883405419, Loss: 2.4166159629821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 91.82086838753058, Tokens per sec: 4189.119972555125, Loss: 2.392437696456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 92.28303214424632, Tokens per sec: 4210.205151315119, Loss: 2.403663158416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 92.30472265489443, Tokens per sec: 4211.194731929708, Loss: 2.401660203933716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 92.2736968296634, Tokens per sec: 4209.779248647718, Loss: 2.4025704860687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 92.21834729594177, Tokens per sec: 4207.254051040064, Loss: 2.39455246925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 91.79192809929859, Tokens per sec: 4187.799637193752, Loss: 2.3742923736572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 91.96991125774728, Tokens per sec: 4195.919717268458, Loss: 2.3907310962677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 92.07877384545262, Tokens per sec: 4200.886327238816, Loss: 2.3874545097351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 91.86701517858016, Tokens per sec: 4191.2253157897285, Loss: 2.3772172927856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 92.08163442426965, Tokens per sec: 4201.016834693885, Loss: 2.3620951175689697 +------------------------------------------------------------------ + eval ppl=10.258813858032227, eval loss=2.328137159347534 +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 91.91399038218691, Tokens per sec: 4193.368453478357, Loss: 2.363520860671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 91.43040483503454, Tokens per sec: 4171.3059538571915, Loss: 2.372114419937134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 92.2682503183832, Tokens per sec: 4209.530763857873, Loss: 2.390418767929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 92.04426962732825, Tokens per sec: 4199.312150128312, Loss: 2.351715564727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 91.55528463261984, Tokens per sec: 4177.003312893542, Loss: 2.3678555488586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 92.13385618515277, Tokens per sec: 4203.3993347220285, Loss: 2.3740458488464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 91.7293559383714, Tokens per sec: 4184.944923514071, Loss: 2.363065242767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 92.12998082105473, Tokens per sec: 4203.2225299778665, Loss: 2.3621938228607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 91.66784529204693, Tokens per sec: 4182.138639043366, Loss: 2.3419697284698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 91.7437614774769, Tokens per sec: 4185.602143736903, Loss: 2.344191551208496 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/optimizer.pt +[rank0]:[2024-10-31 23:49:29,121] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007867839998652926, 'preprocessing_with_comm': 0.0007524220018240158, 'state_converting': 7.508757340001466, : 7.518963059002999}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050, took 31.59s +No files have been modified since last commit. Skipping to prevent empty commit. +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 88.90288249423507, Tokens per sec: 4055.993449141649, Loss: 2.34537672996521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 91.700585250323, Tokens per sec: 4183.632325778455, Loss: 2.3614161014556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 91.60480866330927, Tokens per sec: 4179.26273506767, Loss: 2.350158214569092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 91.25692142417486, Tokens per sec: 4163.391164614815, Loss: 2.357558012008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 91.27677062109932, Tokens per sec: 4164.29673944476, Loss: 2.3394436836242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 92.026107709148, Tokens per sec: 4198.483553584576, Loss: 2.3539419174194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 90.83991835956738, Tokens per sec: 4144.366340549863, Loss: 2.354048013687134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 90.73712535119502, Tokens per sec: 4139.676641443617, Loss: 2.3421099185943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 91.00754129640373, Tokens per sec: 4152.013758886138, Loss: 2.3228492736816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 91.69711705297509, Tokens per sec: 4183.4740970987, Loss: 2.32804274559021 +------------------------------------------------------------------ + eval ppl=10.68118953704834, eval loss=2.3684842586517334 +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 91.60322300424609, Tokens per sec: 4179.190392950156, Loss: 2.3352859020233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 91.17782095213224, Tokens per sec: 4159.782384028317, Loss: 2.342341661453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 91.54928192492189, Tokens per sec: 4176.729452896915, Loss: 2.335371494293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 91.63977810408166, Tokens per sec: 4180.858136911948, Loss: 2.32633900642395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 92.30641608136837, Tokens per sec: 4211.271990692216, Loss: 2.3381786346435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 92.2602551637442, Tokens per sec: 4209.166002964516, Loss: 2.3335485458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 92.15652898883583, Tokens per sec: 4204.433730240286, Loss: 2.338008165359497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 92.23697970855062, Tokens per sec: 4208.1041128306715, Loss: 2.327646493911743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 91.38486834961026, Tokens per sec: 4169.228454439894, Loss: 2.3604958057403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 92.33035925405484, Tokens per sec: 4212.364343930304, Loss: 2.3461477756500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 92.10933290359664, Tokens per sec: 4202.280515325486, Loss: 2.3244080543518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 91.67787527801107, Tokens per sec: 4182.596234525358, Loss: 2.328829050064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 92.06498822123315, Tokens per sec: 4200.257389234133, Loss: 2.3288848400115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 91.98221468884184, Tokens per sec: 4196.48103355564, Loss: 2.353740930557251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 91.74665271371829, Tokens per sec: 4185.734049867795, Loss: 2.316124677658081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 91.40212105735226, Tokens per sec: 4170.015570308562, Loss: 2.3291587829589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 92.35561148806826, Tokens per sec: 4213.516420138196, Loss: 2.3164052963256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 92.38489707670578, Tokens per sec: 4214.852509051576, Loss: 2.3361778259277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 91.56875144431108, Tokens per sec: 4177.617705795865, Loss: 2.354883909225464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 92.45112272956607, Tokens per sec: 4217.873905058436, Loss: 2.35573410987854 +------------------------------------------------------------------ + eval ppl=9.502693176269531, eval loss=2.251575231552124 +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 91.48679107292354, Tokens per sec: 4173.87844875368, Loss: 2.334585189819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 92.29285771546722, Tokens per sec: 4210.6534208356115, Loss: 2.323657751083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 92.25393329148986, Tokens per sec: 4208.877581805038, Loss: 2.3421809673309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 92.09241211744882, Tokens per sec: 4201.50854263074, Loss: 2.3367397785186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 92.19914790102438, Tokens per sec: 4206.37812196074, Loss: 2.321215867996216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 91.99602561268358, Tokens per sec: 4197.111125798514, Loss: 2.3014039993286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 91.82504419927346, Tokens per sec: 4189.310484545274, Loss: 2.315624952316284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 91.67565399591717, Tokens per sec: 4182.494893541031, Loss: 2.3305468559265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 92.57981805868027, Tokens per sec: 4223.745339112964, Loss: 2.319784164428711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 92.15340707238916, Tokens per sec: 4204.291299845441, Loss: 2.3146750926971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 91.4688113456216, Tokens per sec: 4173.0581642577445, Loss: 2.323413610458374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 92.12013307454363, Tokens per sec: 4202.773248759787, Loss: 2.318132162094116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 92.40514194198437, Tokens per sec: 4215.7761353575515, Loss: 2.3226468563079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 92.11342562590448, Tokens per sec: 4202.467236547612, Loss: 2.3084542751312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 91.78262424415126, Tokens per sec: 4187.375169792127, Loss: 2.326049327850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 92.43909263252115, Tokens per sec: 4217.325059020628, Loss: 2.3146369457244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 91.82980103079647, Tokens per sec: 4189.527504252113, Loss: 2.3222689628601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 91.7090736311957, Tokens per sec: 4184.019589006008, Loss: 2.3222453594207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 92.24375781470874, Tokens per sec: 4208.413348632747, Loss: 2.2914655208587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 92.28835479931526, Tokens per sec: 4210.447985444745, Loss: 2.3133351802825928 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/optimizer.pt +[rank0]:[2024-11-01 00:42:19,748] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007582914000522578, 'preprocessing_with_comm': 0.0006532900006277487, 'state_converting': 7.623732492000272, : 7.633553055999073}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100, took 31.82s + eval ppl=10.825410842895508, eval loss=2.3818962574005127 +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 90.99313580278945, Tokens per sec: 4151.3565407387405, Loss: 2.3037986755371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 91.77881916378202, Tokens per sec: 4187.20157158453, Loss: 2.323740005493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 92.03508952306814, Tokens per sec: 4198.893328581736, Loss: 2.3053646087646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 92.54937085611004, Tokens per sec: 4222.356254184503, Loss: 2.2983815670013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 91.88536300388208, Tokens per sec: 4192.062393926486, Loss: 2.3388421535491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 92.00263547372758, Tokens per sec: 4197.412685796819, Loss: 2.3148558139801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 91.68735127468, Tokens per sec: 4183.028555495555, Loss: 2.3119547367095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 92.19798534973485, Tokens per sec: 4206.325083180868, Loss: 2.3136298656463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 91.93199100458925, Tokens per sec: 4194.18969126611, Loss: 2.3064143657684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 91.64408700567625, Tokens per sec: 4181.0547208263315, Loss: 2.3451364040374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 92.51059651567587, Tokens per sec: 4220.587262376999, Loss: 2.311394453048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 91.95127585674808, Tokens per sec: 4195.069519139282, Loss: 2.31459379196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 92.3719355248378, Tokens per sec: 4214.261167488824, Loss: 2.3184688091278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 92.00617559805616, Tokens per sec: 4197.574195982799, Loss: 2.2972352504730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 92.00465420809913, Tokens per sec: 4197.504785998261, Loss: 2.3222057819366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 91.4822689007822, Tokens per sec: 4173.672134851739, Loss: 2.3114943504333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 92.59163867825964, Tokens per sec: 4224.284628213988, Loss: 2.3164446353912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 91.29255482835256, Tokens per sec: 4165.016858291561, Loss: 2.307446002960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 92.60779217953355, Tokens per sec: 4225.021595267355, Loss: 2.298893451690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 91.93344260474775, Tokens per sec: 4194.255917248535, Loss: 2.287914752960205 +------------------------------------------------------------------ + eval ppl=10.813911437988281, eval loss=2.380833387374878 +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 92.32453405932038, Tokens per sec: 4212.098582562168, Loss: 2.3093674182891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 91.63691144262799, Tokens per sec: 4180.727351950305, Loss: 2.289240598678589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 92.20475667042929, Tokens per sec: 4206.634009411471, Loss: 2.2991487979888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 91.76020578747746, Tokens per sec: 4186.35237828236, Loss: 2.2783093452453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 91.4303172010517, Tokens per sec: 4171.301955754427, Loss: 2.2834813594818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 91.99107325964059, Tokens per sec: 4196.8851858634225, Loss: 2.2828686237335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 91.9306886097973, Tokens per sec: 4194.130272441926, Loss: 2.276123285293579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 92.23503057167518, Tokens per sec: 4208.015187857977, Loss: 2.3085813522338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 92.50789724936396, Tokens per sec: 4220.464114441054, Loss: 2.297600746154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 91.91217914727568, Tokens per sec: 4193.285819971674, Loss: 2.279604911804199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 91.86142048892185, Tokens per sec: 4190.970070695674, Loss: 2.2724416255950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 91.38277957560932, Tokens per sec: 4169.13315883837, Loss: 2.3163933753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 92.21638444342574, Tokens per sec: 4207.16450032221, Loss: 2.29360032081604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 91.83200273991855, Tokens per sec: 4189.627952263758, Loss: 2.286407232284546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 92.50827591082057, Tokens per sec: 4220.481390015754, Loss: 2.2889978885650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 92.26152788452747, Tokens per sec: 4209.224067978998, Loss: 2.2878329753875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 92.16040272653244, Tokens per sec: 4204.610460783552, Loss: 2.2676968574523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 92.08766958560004, Tokens per sec: 4201.292175314277, Loss: 2.2701947689056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 91.94637374162029, Tokens per sec: 4194.8458711957355, Loss: 2.282926082611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 92.46817536808165, Tokens per sec: 4218.651893220004, Loss: 2.2828445434570312 +------------------------------------------------------------------ + eval ppl=10.366499900817871, eval loss=2.3385794162750244 +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 91.68485892128126, Tokens per sec: 4182.914847494481, Loss: 2.2865779399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 92.5101999928558, Tokens per sec: 4220.569171917887, Loss: 2.297614336013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 91.99908744423368, Tokens per sec: 4197.250814955503, Loss: 2.2612478733062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 91.76482490611922, Tokens per sec: 4186.563115149672, Loss: 2.2781946659088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 92.61066744961703, Tokens per sec: 4225.152772978312, Loss: 2.291734218597412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 92.01891025604951, Tokens per sec: 4198.155186024393, Loss: 2.278754711151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 91.68185087543623, Tokens per sec: 4182.777612188945, Loss: 2.2940468788146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 92.12802341073912, Tokens per sec: 4203.133227548129, Loss: 2.301339864730835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 92.42310482484261, Tokens per sec: 4216.595651363744, Loss: 2.2578916549682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 91.51330856994117, Tokens per sec: 4175.088249731709, Loss: 2.2808094024658203 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/model.pt +[rank0]:[2024-11-01 01:34:59,918] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007656737998331664, 'preprocessing_with_comm': 0.0006429499990190379, 'state_converting': 7.46211816000141, : 7.472006348001742}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150, took 31.62s +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 90.01826906413821, Tokens per sec: 4106.880445084453, Loss: 2.2828195095062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 91.68059266360875, Tokens per sec: 4182.720209112838, Loss: 2.270493745803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 92.4332922454957, Tokens per sec: 4217.060429448248, Loss: 2.285165309906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 91.43340311744358, Tokens per sec: 4171.442743727987, Loss: 2.276259183883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 92.62084541638545, Tokens per sec: 4225.6171197506565, Loss: 2.2517964839935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 91.93829087057351, Tokens per sec: 4194.477108439161, Loss: 2.2770261764526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 92.05862635155499, Tokens per sec: 4199.967143282428, Loss: 2.257293462753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 91.68365222249604, Tokens per sec: 4182.859794584714, Loss: 2.2709906101226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 92.02851966856693, Tokens per sec: 4198.593593791691, Loss: 2.264690399169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 91.61894645653013, Tokens per sec: 4179.907739988549, Loss: 2.286133289337158 +------------------------------------------------------------------ + eval ppl=9.286602973937988, eval loss=2.2285728454589844 +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 92.33760526226894, Tokens per sec: 4212.694926708098, Loss: 2.2776758670806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 91.54823727789828, Tokens per sec: 4176.681793233183, Loss: 2.2665393352508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 92.19125298359737, Tokens per sec: 4206.017934164045, Loss: 2.274078845977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 92.16717632908998, Tokens per sec: 4204.919491119012, Loss: 2.2633798122406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 91.57722421712313, Tokens per sec: 4178.004256940862, Loss: 2.2617557048797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 91.77371270067877, Tokens per sec: 4186.968600725611, Loss: 2.2750332355499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 92.22761013998802, Tokens per sec: 4207.676647402724, Loss: 2.2778356075286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 91.73590851365118, Tokens per sec: 4185.24386997862, Loss: 2.2916882038116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 92.21615171701536, Tokens per sec: 4207.153882704767, Loss: 2.2745590209960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 92.04371506204409, Tokens per sec: 4199.28684933832, Loss: 2.2483649253845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 91.7947027785045, Tokens per sec: 4187.9262256728525, Loss: 2.271738052368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 92.4790007013486, Tokens per sec: 4219.145774628387, Loss: 2.2501258850097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 91.74280523391953, Tokens per sec: 4185.55851728188, Loss: 2.2530858516693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 91.84518248203153, Tokens per sec: 4190.2292482642015, Loss: 2.2501728534698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 91.44600798429606, Tokens per sec: 4172.017812341587, Loss: 2.246927499771118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 91.90374931450773, Tokens per sec: 4192.901227869315, Loss: 2.2606890201568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 91.66092466851698, Tokens per sec: 4181.822901207719, Loss: 2.2603511810302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 92.41475813540026, Tokens per sec: 4216.214852488046, Loss: 2.2730226516723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 92.05441316488165, Tokens per sec: 4199.774926145395, Loss: 2.2686007022857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 92.21566374396275, Tokens per sec: 4207.1316200351075, Loss: 2.275812864303589 +------------------------------------------------------------------ + eval ppl=10.66044807434082, eval loss=2.3665404319763184 +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 91.761273245102, Tokens per sec: 4186.401078628299, Loss: 2.266066789627075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 92.45353736963337, Tokens per sec: 4217.984067564085, Loss: 2.268435001373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 92.64737084847945, Tokens per sec: 4226.827282748644, Loss: 2.221801996231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 92.41689254881003, Tokens per sec: 4216.312230284636, Loss: 2.2657155990600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 92.47621595805734, Tokens per sec: 4219.018726998096, Loss: 2.250221014022827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 91.72281972388734, Tokens per sec: 4184.6467234739175, Loss: 2.2491230964660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 92.36880871819484, Tokens per sec: 4214.118513989807, Loss: 2.259385108947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 91.94334862639155, Tokens per sec: 4194.707857138076, Loss: 2.2601840496063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 91.97242378650014, Tokens per sec: 4196.034345724549, Loss: 2.2692089080810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 91.93612793150342, Tokens per sec: 4194.378429223673, Loss: 2.2679221630096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 92.16894565460899, Tokens per sec: 4205.000212604223, Loss: 2.226013422012329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 92.80703933226219, Tokens per sec: 4234.111797109573, Loss: 2.256215810775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 91.41478159014044, Tokens per sec: 4170.5931785548955, Loss: 2.244910955429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 92.60054863704008, Tokens per sec: 4224.6911249825025, Loss: 2.2670016288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 92.00136687896956, Tokens per sec: 4197.35480902293, Loss: 2.263120174407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 92.18372854520773, Tokens per sec: 4205.67464863763, Loss: 2.2313969135284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 92.3758827023929, Tokens per sec: 4214.441248559954, Loss: 2.25089430809021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 91.51691259699945, Tokens per sec: 4175.252675335568, Loss: 2.2755610942840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 91.89749627689508, Tokens per sec: 4192.615947135074, Loss: 2.267009735107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 91.89320204624069, Tokens per sec: 4192.420032549239, Loss: 2.24961256980896 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/model.pt +[rank0]:[2024-11-01 02:27:40,011] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007564978994196281, 'preprocessing_with_comm': 0.0006565410003531724, 'state_converting': 7.416041630000109, : 7.4258518280039425}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200, took 31.45s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050 + eval ppl=8.520344734191895, eval loss=2.1424567699432373 +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 90.7120301872639, Tokens per sec: 4138.5317312038815, Loss: 2.2414894104003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 91.35039554159896, Tokens per sec: 4167.655710344938, Loss: 2.2213101387023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 91.71935819768065, Tokens per sec: 4184.48879914993, Loss: 2.259525775909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 91.62322995400645, Tokens per sec: 4180.1031648973485, Loss: 2.256309986114502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 91.96178896162414, Tokens per sec: 4195.549155831718, Loss: 2.21846866607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 92.01517741062669, Tokens per sec: 4197.984883373265, Loss: 2.255038022994995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 92.30044100766614, Tokens per sec: 4210.999391434301, Loss: 2.2241694927215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 91.94290556469647, Tokens per sec: 4194.6876434477845, Loss: 2.2176826000213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 90.77902133628635, Tokens per sec: 4141.588051246183, Loss: 2.2627179622650146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 92.32946109147356, Tokens per sec: 4212.323367288786, Loss: 2.2364094257354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 91.98991350145278, Tokens per sec: 4196.832274512514, Loss: 2.2393527030944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 91.66736661096338, Tokens per sec: 4182.116800298832, Loss: 2.2403860092163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 92.25712119018633, Tokens per sec: 4209.023022491174, Loss: 2.243037223815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 91.99798117038179, Tokens per sec: 4197.200343706762, Loss: 2.241992950439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 92.07562362517264, Tokens per sec: 4200.742605545431, Loss: 2.2103264331817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 91.8840526732956, Tokens per sec: 4192.00261304958, Loss: 2.230509042739868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 91.86114845664666, Tokens per sec: 4190.957659836821, Loss: 2.2617855072021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 91.93142473562804, Tokens per sec: 4194.163856522256, Loss: 2.2328295707702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 91.88828049055418, Tokens per sec: 4192.195497674059, Loss: 2.2246241569519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 91.985185056927, Tokens per sec: 4196.616549898384, Loss: 2.2362020015716553 +------------------------------------------------------------------ + eval ppl=8.672445297241211, eval loss=2.1601507663726807 +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 92.15666147021474, Tokens per sec: 4204.439774404321, Loss: 2.2328479290008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 92.02104655835196, Tokens per sec: 4198.252649997449, Loss: 2.2191450595855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 91.38781344440473, Tokens per sec: 4169.362817745766, Loss: 2.2304940223693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 92.28109400263136, Tokens per sec: 4210.11672797638, Loss: 2.2322704792022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 91.53317735180391, Tokens per sec: 4175.99471808028, Loss: 2.2386648654937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 91.65416598291455, Tokens per sec: 4181.514551424684, Loss: 2.22835636138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 92.1304064034547, Tokens per sec: 4203.241946214753, Loss: 2.2080724239349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 92.222225525638, Tokens per sec: 4207.430986520679, Loss: 2.2267770767211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 92.20576694085234, Tokens per sec: 4206.680100720356, Loss: 2.2419116497039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 91.40394874633186, Tokens per sec: 4170.098954494995, Loss: 2.2291319370269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 92.54695935637622, Tokens per sec: 4222.246234949484, Loss: 2.2214882373809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 91.78688214286628, Tokens per sec: 4187.569426815191, Loss: 2.261282205581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 92.32154334858494, Tokens per sec: 4211.962138131898, Loss: 2.2384588718414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 91.99876372665696, Tokens per sec: 4197.236046071372, Loss: 2.2256808280944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 91.96132743809028, Tokens per sec: 4195.528099861681, Loss: 2.238142728805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 92.02110461941655, Tokens per sec: 4198.255298902535, Loss: 2.200291872024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 92.1477528363321, Tokens per sec: 4204.03333808128, Loss: 2.2512378692626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 91.77548906765398, Tokens per sec: 4187.049643461376, Loss: 2.230604410171509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 92.42051401084316, Tokens per sec: 4216.477451319889, Loss: 2.2311224937438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 92.09013893287752, Tokens per sec: 4201.404833712972, Loss: 2.204885721206665 +------------------------------------------------------------------ + eval ppl=9.687769889831543, eval loss=2.270864248275757 +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 92.32293708378405, Tokens per sec: 4212.025724156096, Loss: 2.2287001609802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 91.96221676315392, Tokens per sec: 4195.568673311394, Loss: 2.2278037071228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 92.27519449295085, Tokens per sec: 4209.847576156483, Loss: 2.1991934776306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 92.07119582536926, Tokens per sec: 4200.540597168505, Loss: 2.225112199783325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 91.64231575531231, Tokens per sec: 4180.973911524413, Loss: 2.2117772102355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 92.27142245546932, Tokens per sec: 4209.675485456096, Loss: 2.2276573181152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 92.48633230993536, Tokens per sec: 4219.480262730069, Loss: 2.245633602142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 91.05746460879418, Tokens per sec: 4154.291397387097, Loss: 2.2249438762664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 92.17108544757077, Tokens per sec: 4205.0978358307375, Loss: 2.221712350845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 92.05859367844594, Tokens per sec: 4199.965652645537, Loss: 2.2407333850860596 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/model.pt +[rank0]:[2024-11-01 03:20:23,447] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007658103000721894, 'preprocessing_with_comm': 0.0006424499952117912, 'state_converting': 7.330401040999277, : 7.340449932999036}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250, took 31.57s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000100 +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 89.33674318013553, Tokens per sec: 4075.787364147346, Loss: 2.218083143234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 92.6713949271217, Tokens per sec: 4227.923327138442, Loss: 2.221395254135132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 91.95680396594999, Tokens per sec: 4195.321726650221, Loss: 2.230471134185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 92.02984530549261, Tokens per sec: 4198.6540729857925, Loss: 2.233362913131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 91.88291941348548, Tokens per sec: 4191.9509106273745, Loss: 2.227612257003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 92.035880473697, Tokens per sec: 4198.929413919795, Loss: 2.253919839859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 91.75540837310395, Tokens per sec: 4186.133507074523, Loss: 2.2292912006378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 91.42504315807626, Tokens per sec: 4171.061339441899, Loss: 2.2403712272644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 91.40965891208202, Tokens per sec: 4170.359467925228, Loss: 2.2341415882110596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 92.53148374751491, Tokens per sec: 4221.5401952081165, Loss: 2.218763828277588 +------------------------------------------------------------------ + eval ppl=9.713004112243652, eval loss=2.273465633392334 +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 92.45071600040914, Tokens per sec: 4217.855348958232, Loss: 2.2122159004211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 91.72298040224894, Tokens per sec: 4184.654054061675, Loss: 2.232372999191284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 92.14887750106583, Tokens per sec: 4204.0846483725145, Loss: 2.215089797973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 91.84300931142667, Tokens per sec: 4190.130102258021, Loss: 2.216668128967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 91.31665022456951, Tokens per sec: 4166.116156384812, Loss: 2.2072529792785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 92.60090192490352, Tokens per sec: 4224.707242944256, Loss: 2.238825798034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 91.57513086463112, Tokens per sec: 4177.908752456012, Loss: 2.203585624694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 91.99908894083934, Tokens per sec: 4197.250883234761, Loss: 2.216402053833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 92.34938897839338, Tokens per sec: 4213.232532172245, Loss: 2.2242071628570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 92.06132534465479, Tokens per sec: 4200.090278753684, Loss: 2.215545415878296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 92.33972376107187, Tokens per sec: 4212.791578436654, Loss: 2.1964612007141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 91.48238401496775, Tokens per sec: 4173.677386676758, Loss: 2.1971521377563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 92.07204940291085, Tokens per sec: 4200.579539718172, Loss: 2.243417501449585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 92.47298373252872, Tokens per sec: 4218.871264000245, Loss: 2.2309606075286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 91.56764579389565, Tokens per sec: 4177.5672629900055, Loss: 2.191546678543091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 91.62605622573201, Tokens per sec: 4180.232107168766, Loss: 2.2105472087860107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 91.65517201907944, Tokens per sec: 4181.560449555092, Loss: 2.1812007427215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 92.39582288575667, Tokens per sec: 4215.3509744409275, Loss: 2.2171690464019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 92.05614432380919, Tokens per sec: 4199.853906365974, Loss: 2.209920644760132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 92.25769409207778, Tokens per sec: 4209.049159847506, Loss: 2.2036774158477783 +------------------------------------------------------------------ + eval ppl=8.381685256958008, eval loss=2.126049041748047 +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 91.65426055045505, Tokens per sec: 4181.518865855375, Loss: 2.2134363651275635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 92.08737188245955, Tokens per sec: 4201.27859328011, Loss: 2.2126309871673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 91.41460805312295, Tokens per sec: 4170.585261319971, Loss: 2.225914478302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 92.19801113852952, Tokens per sec: 4206.3262597364455, Loss: 2.201995611190796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 91.56538653911868, Tokens per sec: 4177.464189587678, Loss: 2.1855459213256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 91.84186429741744, Tokens per sec: 4190.077863576999, Loss: 2.203021764755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 92.38113361892174, Tokens per sec: 4214.68080978053, Loss: 2.2030954360961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 91.61873795027637, Tokens per sec: 4179.898227361112, Loss: 2.1924893856048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 92.61539266232458, Tokens per sec: 4225.368350148028, Loss: 2.1858811378479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 91.61375269789137, Tokens per sec: 4179.670786467761, Loss: 2.205092668533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 91.48162802456194, Tokens per sec: 4173.642896320005, Loss: 2.2049193382263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 91.93264065892352, Tokens per sec: 4194.219330279492, Loss: 2.2099223136901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 92.6755576408106, Tokens per sec: 4228.1132415594275, Loss: 2.2148947715759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 91.74465208744988, Tokens per sec: 4185.642775807706, Loss: 2.2081565856933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 92.26318027588984, Tokens per sec: 4209.299454607116, Loss: 2.2084507942199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 91.65089453971352, Tokens per sec: 4181.365299209004, Loss: 2.185931444168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 91.70015690719364, Tokens per sec: 4183.6127835895195, Loss: 2.200767755508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 91.8242076460957, Tokens per sec: 4189.272318693801, Loss: 2.219517230987549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 91.96329923819638, Tokens per sec: 4195.618058793146, Loss: 2.1961965560913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 92.13765352224075, Tokens per sec: 4203.572579659886, Loss: 2.1860969066619873 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/model.pt +[rank0]:[2024-11-01 04:13:05,843] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0076953739990131, 'preprocessing_with_comm': 0.000687757994455751, 'state_converting': 7.257466220995411, : 7.267425560996344}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300, took 31.02s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000150 + eval ppl=9.555915832519531, eval loss=2.2571604251861572 +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 90.87065810314239, Tokens per sec: 4145.768772001696, Loss: 2.2073752880096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 91.97338619791512, Tokens per sec: 4196.07825357416, Loss: 2.2044055461883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 91.9021701476359, Tokens per sec: 4192.829181943384, Loss: 2.2112016677856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 91.98047378241361, Tokens per sec: 4196.401608627333, Loss: 2.195652484893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 91.70822219204419, Tokens per sec: 4183.9807440155655, Loss: 2.1934609413146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 92.43487551750202, Tokens per sec: 4217.13266266168, Loss: 2.214021682739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 92.23941482317005, Tokens per sec: 4208.2152094415715, Loss: 2.1761398315429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 91.83083627997404, Tokens per sec: 4189.57473516032, Loss: 2.1882803440093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 91.78792082346351, Tokens per sec: 4187.616814274179, Loss: 2.1734015941619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 91.62339713348504, Tokens per sec: 4180.110792083902, Loss: 2.228088617324829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 91.65474004989844, Tokens per sec: 4181.54074193573, Loss: 2.1801586151123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 92.0471145408902, Tokens per sec: 4199.44194289146, Loss: 2.187520980834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 91.5468491972478, Tokens per sec: 4176.618465185011, Loss: 2.186573028564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 92.11926343191121, Tokens per sec: 4202.733573276594, Loss: 2.2165958881378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 92.24921322416434, Tokens per sec: 4208.66223938177, Loss: 2.2045693397521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 91.52026610544786, Tokens per sec: 4175.405671593021, Loss: 2.1918702125549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 91.56244361439995, Tokens per sec: 4177.329925286616, Loss: 2.207547903060913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 92.480568736218, Tokens per sec: 4219.217312681821, Loss: 2.175124406814575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 92.13307926030073, Tokens per sec: 4203.363889278395, Loss: 2.1761984825134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 91.8946286273536, Tokens per sec: 4192.485117094153, Loss: 2.1708898544311523 +------------------------------------------------------------------ + eval ppl=9.341447830200195, eval loss=2.2344613075256348 +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 91.25456076695228, Tokens per sec: 4163.283464954671, Loss: 2.179936170578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 91.96596498245952, Tokens per sec: 4195.739677361241, Loss: 2.1778674125671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 91.88992322853649, Tokens per sec: 4192.270443888488, Loss: 2.185965061187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 91.95180837342203, Tokens per sec: 4195.093814011176, Loss: 2.1886684894561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 92.16757187967163, Tokens per sec: 4204.937537221938, Loss: 2.1834120750427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 92.47924245594757, Tokens per sec: 4219.156804136567, Loss: 2.167525291442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 92.29420273229638, Tokens per sec: 4210.714784194089, Loss: 2.1825053691864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 91.73381762700932, Tokens per sec: 4185.148477992619, Loss: 2.18440842628479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 92.12557835812024, Tokens per sec: 4203.021677538425, Loss: 2.176379442214966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 91.98914091643132, Tokens per sec: 4196.797027063861, Loss: 2.180274724960327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 92.22684980709448, Tokens per sec: 4207.641958929765, Loss: 2.1959588527679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 92.31039080376472, Tokens per sec: 4211.453328434586, Loss: 2.1706957817077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 91.55373860832722, Tokens per sec: 4176.932779023018, Loss: 2.1797499656677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 91.52802447987438, Tokens per sec: 4175.759629923359, Loss: 2.1850943565368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 91.87997572544816, Tokens per sec: 4191.816611501624, Loss: 2.168377637863159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 91.99155513029649, Tokens per sec: 4196.907170124999, Loss: 2.1665730476379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 91.75325637965194, Tokens per sec: 4186.035327228165, Loss: 2.190627336502075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 91.74713428973227, Tokens per sec: 4185.756020686998, Loss: 2.2009990215301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 92.26397640394642, Tokens per sec: 4209.3357761536345, Loss: 2.183021068572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 91.7129977923216, Tokens per sec: 4184.198620004481, Loss: 2.1929540634155273 +------------------------------------------------------------------ + eval ppl=8.205130577087402, eval loss=2.104759693145752 +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 92.27920380291904, Tokens per sec: 4210.030491878791, Loss: 2.1664769649505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 91.79339238214166, Tokens per sec: 4187.866441795049, Loss: 2.1895761489868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 92.90766347413462, Tokens per sec: 4238.702546574715, Loss: 2.154507875442505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 91.70123217188281, Tokens per sec: 4183.661840115169, Loss: 2.176384210586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 91.90618564775409, Tokens per sec: 4193.012380077319, Loss: 2.178476572036743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 91.61707972102046, Tokens per sec: 4179.822574392271, Loss: 2.1822898387908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 92.75868472346524, Tokens per sec: 4231.905727171075, Loss: 2.1919336318969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 92.33351806237052, Tokens per sec: 4212.508457433444, Loss: 2.1600186824798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 91.74356804227855, Tokens per sec: 4185.593318692368, Loss: 2.1713879108428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 92.48009320410712, Tokens per sec: 4219.195617602061, Loss: 2.2021656036376953 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/optimizer.pt +[rank0]:[2024-11-01 05:05:47,729] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007571948997792788, 'preprocessing_with_comm': 0.0006456420014728792, 'state_converting': 7.296920556000259, : 7.306717762003245}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350, took 31.12s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000200 +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 91.18371399684472, Tokens per sec: 4160.051240898632, Loss: 2.1510722637176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 91.52025742443595, Tokens per sec: 4175.405275541437, Loss: 2.171375036239624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 92.2330456096184, Tokens per sec: 4207.924628442206, Loss: 2.1644816398620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 91.57918734466386, Tokens per sec: 4178.093820206098, Loss: 2.154766321182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 91.53442771268334, Tokens per sec: 4176.051762974594, Loss: 2.179917812347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 91.93056052279708, Tokens per sec: 4194.1244287615045, Loss: 2.166987657546997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 91.91229080292231, Tokens per sec: 4193.290914008656, Loss: 2.1826462745666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 92.19926545899759, Tokens per sec: 4206.383485278057, Loss: 2.154609203338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 91.33341930941565, Tokens per sec: 4166.881208049907, Loss: 2.202677011489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 91.83646948688965, Tokens per sec: 4189.8317375173465, Loss: 2.154871702194214 +------------------------------------------------------------------ + eval ppl=8.221978187561035, eval loss=2.1068108081817627 +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 91.86949254146948, Tokens per sec: 4191.33833988263, Loss: 2.167020320892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 92.34097744536375, Tokens per sec: 4212.848774954164, Loss: 2.174192428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 91.65847669050646, Tokens per sec: 4181.711217733621, Loss: 2.1616628170013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 92.09590310301735, Tokens per sec: 4201.6678109716495, Loss: 2.140953302383423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 92.23649753829726, Tokens per sec: 4208.082114900636, Loss: 2.169649839401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 91.78393776783513, Tokens per sec: 4187.435096346897, Loss: 2.1575369834899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 92.34079089621214, Tokens per sec: 4212.840264069975, Loss: 2.1622087955474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 91.9888476370066, Tokens per sec: 4196.783646851745, Loss: 2.15415096282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 91.71420125503293, Tokens per sec: 4184.253525275677, Loss: 2.142545461654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 92.42851339986034, Tokens per sec: 4216.842405397198, Loss: 2.162916421890259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 91.54047345578508, Tokens per sec: 4176.327586364429, Loss: 2.17549729347229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 91.98888007082498, Tokens per sec: 4196.785126571538, Loss: 2.1551787853240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 91.84625546854856, Tokens per sec: 4190.278200853386, Loss: 2.1770942211151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 92.17035913266074, Tokens per sec: 4205.064699351531, Loss: 2.152981758117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 91.89299112553984, Tokens per sec: 4192.4104097681, Loss: 2.147063732147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 91.50277391094023, Tokens per sec: 4174.607629680953, Loss: 2.1675596237182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 92.4463969391574, Tokens per sec: 4217.65830153241, Loss: 2.1731417179107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 92.38688083963808, Tokens per sec: 4214.943013759998, Loss: 2.1850881576538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 91.83210206473012, Tokens per sec: 4189.63248373421, Loss: 2.1387486457824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 92.56030985566552, Tokens per sec: 4222.855321360868, Loss: 2.1572537422180176 +------------------------------------------------------------------ + eval ppl=9.214385032653809, eval loss=2.2207658290863037 +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 91.61429064453758, Tokens per sec: 4179.695329069888, Loss: 2.145348072052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 92.08717001284096, Tokens per sec: 4201.269383434179, Loss: 2.154294729232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 92.49281929809858, Tokens per sec: 4219.776216930389, Loss: 2.1704318523406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 91.57358664278627, Tokens per sec: 4177.838300818101, Loss: 2.162503480911255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 91.70355217172565, Tokens per sec: 4183.767684874085, Loss: 2.1501338481903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 92.80412249099187, Tokens per sec: 4233.978722807, Loss: 2.1513760089874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 91.6999655257719, Tokens per sec: 4183.604052243909, Loss: 2.1499757766723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 91.8432420897545, Tokens per sec: 4190.140722244081, Loss: 2.1838674545288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 91.69634036408473, Tokens per sec: 4183.438662420286, Loss: 2.1436705589294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 92.72247213992652, Tokens per sec: 4230.25360974261, Loss: 2.149035930633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 92.20999558732, Tokens per sec: 4206.873023175627, Loss: 2.1726267337799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 92.38084978092722, Tokens per sec: 4214.667860312367, Loss: 2.1646037101745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 92.31894639883903, Tokens per sec: 4211.843658158453, Loss: 2.1722562313079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 91.58793694859158, Tokens per sec: 4178.4930011462175, Loss: 2.140427589416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 92.46420261758738, Tokens per sec: 4218.470645441217, Loss: 2.186835527420044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 91.93866677174574, Tokens per sec: 4194.4942580821125, Loss: 2.1526999473571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 92.07578159598691, Tokens per sec: 4200.74981260745, Loss: 2.165884256362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 92.09686777151514, Tokens per sec: 4201.711821795571, Loss: 2.1581764221191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 91.98317789506493, Tokens per sec: 4196.524977666578, Loss: 2.1747119426727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 92.48405603136925, Tokens per sec: 4219.376412655768, Loss: 2.125241279602051 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/optimizer.pt +[rank0]:[2024-11-01 05:58:27,460] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007679086003918201, 'preprocessing_with_comm': 0.0006524109994643368, 'state_converting': 7.324405282000953, : 7.334303411997098}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400, took 31.11s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000250 + eval ppl=9.16974925994873, eval loss=2.215909957885742 +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 89.75153144984588, Tokens per sec: 4094.711143191688, Loss: 2.1815855503082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 91.3122021510408, Tokens per sec: 4165.913223064906, Loss: 2.163729429244995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 92.72466805788653, Tokens per sec: 4230.353793545551, Loss: 2.150907516479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 91.75610071781601, Tokens per sec: 4186.165093740083, Loss: 2.137514114379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 92.05543799640411, Tokens per sec: 4199.821681771589, Loss: 2.1437413692474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 92.14076304218305, Tokens per sec: 4203.714444492185, Loss: 2.157428026199341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 91.8816856293478, Tokens per sec: 4191.894622009506, Loss: 2.152080774307251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 91.65993462837851, Tokens per sec: 4181.777732859929, Loss: 2.1349241733551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 91.78607716674199, Tokens per sec: 4187.532701595544, Loss: 2.1531081199645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 91.49813911077, Tokens per sec: 4174.396177379283, Loss: 2.1559805870056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 91.87226846326041, Tokens per sec: 4191.46498505187, Loss: 2.165817975997925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 91.81220823206901, Tokens per sec: 4188.724872499479, Loss: 2.1440980434417725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 91.98667296914209, Tokens per sec: 4196.684432536472, Loss: 2.1566710472106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 91.42921189566584, Tokens per sec: 4171.251528689763, Loss: 2.132246494293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 92.9649610894768, Tokens per sec: 4241.3166210114405, Loss: 2.151445150375366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 91.95830875931328, Tokens per sec: 4195.390379452611, Loss: 2.1482114791870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 91.92287219516888, Tokens per sec: 4193.773666158313, Loss: 2.150151014328003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 91.58687410625936, Tokens per sec: 4178.444511362579, Loss: 2.095876932144165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 92.3427579809369, Tokens per sec: 4212.930007872805, Loss: 2.1253128051757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 91.6570762555602, Tokens per sec: 4181.6473260485745, Loss: 2.1472361087799072 +------------------------------------------------------------------ + eval ppl=10.115959167480469, eval loss=2.3141143321990967 +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 91.94842890143313, Tokens per sec: 4194.939633225762, Loss: 2.1205170154571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 91.62162563485303, Tokens per sec: 4180.029971455314, Loss: 2.130422353744507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 92.46125775179972, Tokens per sec: 4218.336292583264, Loss: 2.132560968399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 91.7858470261224, Tokens per sec: 4187.522201948972, Loss: 2.144401788711548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 92.14913870265843, Tokens per sec: 4204.096565105907, Loss: 2.1436643600463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 92.03664932699297, Tokens per sec: 4198.9644911168925, Loss: 2.1495869159698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 92.75998637133947, Tokens per sec: 4231.965111918824, Loss: 2.137444496154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 91.96173842583921, Tokens per sec: 4195.546850250539, Loss: 2.135092258453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 92.15370521740611, Tokens per sec: 4204.304902039226, Loss: 2.1504063606262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 92.23631988528466, Tokens per sec: 4208.074009882721, Loss: 2.156208038330078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 91.25049057384302, Tokens per sec: 4163.097771576406, Loss: 2.1728224754333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 91.556369785692, Tokens per sec: 4177.052820554357, Loss: 2.143507242202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 91.48314754375167, Tokens per sec: 4173.712220955005, Loss: 2.145502805709839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 92.76093493435422, Tokens per sec: 4232.008387966416, Loss: 2.156403064727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 91.80726612535206, Tokens per sec: 4188.499400029896, Loss: 2.1367902755737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 91.88266393168628, Tokens per sec: 4191.939254846652, Loss: 2.1092097759246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 92.63892446766383, Tokens per sec: 4226.441935678943, Loss: 2.122282028198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 92.29460007300077, Tokens per sec: 4210.732911967326, Loss: 2.1258225440979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 91.20768794483618, Tokens per sec: 4161.144998190582, Loss: 2.1251168251037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 91.55947912343312, Tokens per sec: 4177.194677074138, Loss: 2.1485214233398438 +------------------------------------------------------------------ + eval ppl=7.979638576507568, eval loss=2.0768930912017822 +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 92.05941771121138, Tokens per sec: 4200.0032472814455, Loss: 2.1407392024993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 91.76276393331891, Tokens per sec: 4186.469087915207, Loss: 2.130162239074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 91.54485740344958, Tokens per sec: 4176.5275940864485, Loss: 2.101426601409912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 91.58504544723903, Tokens per sec: 4178.361082920227, Loss: 2.1247503757476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 91.83519520304449, Tokens per sec: 4189.773601191691, Loss: 2.133470058441162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 91.86896701916761, Tokens per sec: 4191.314364113185, Loss: 2.1198649406433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 91.74652997888315, Tokens per sec: 4185.728450367831, Loss: 2.1161203384399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 92.18766084508417, Tokens per sec: 4205.8540509482345, Loss: 2.122058153152466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 91.76757531507484, Tokens per sec: 4186.688596353357, Loss: 2.126558303833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 91.27778832786025, Tokens per sec: 4164.343170019783, Loss: 2.1416079998016357 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/model.pt +[rank0]:[2024-11-01 06:51:12,347] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007589202999952249, 'preprocessing_with_comm': 0.0006428499982575886, 'state_converting': 7.347693880998122, : 7.357520003999525}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450, took 32.10s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000300 +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 90.10245984701042, Tokens per sec: 4110.721459618824, Loss: 2.127952814102173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 91.87824582148018, Tokens per sec: 4191.737688535743, Loss: 2.1381945610046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 91.63498843523844, Tokens per sec: 4180.639619076456, Loss: 2.1259074211120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 91.28894258344404, Tokens per sec: 4164.852057766824, Loss: 2.148216724395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 91.97253516205672, Tokens per sec: 4196.039426983055, Loss: 2.1305789947509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 91.86352259164683, Tokens per sec: 4191.065974390162, Loss: 2.127319097518921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 91.92256058610819, Tokens per sec: 4193.759449697909, Loss: 2.1311869621276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 91.26006661631615, Tokens per sec: 4163.534656910717, Loss: 2.1176812648773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 91.3588692771969, Tokens per sec: 4168.0423054148705, Loss: 2.1447880268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 91.95236710793444, Tokens per sec: 4195.1193050128, Loss: 2.14359450340271 +------------------------------------------------------------------ + eval ppl=8.140215873718262, eval loss=2.0968167781829834 +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 92.25746059032973, Tokens per sec: 4209.03850685703, Loss: 2.15177059173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 91.61281573527313, Tokens per sec: 4179.62803966209, Loss: 2.1303603649139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 91.57938150174854, Tokens per sec: 4178.102678185069, Loss: 2.1344809532165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 91.92253697270552, Tokens per sec: 4193.758372389684, Loss: 2.102130174636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 92.13452733356992, Tokens per sec: 4203.429954354452, Loss: 2.119270086288452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 91.1888978024618, Tokens per sec: 4160.2877403352595, Loss: 2.1475627422332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 91.78217353064383, Tokens per sec: 4187.354607005158, Loss: 2.1035385131835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 91.76268183633604, Tokens per sec: 4186.465342425574, Loss: 2.139525890350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 92.1229013903116, Tokens per sec: 4202.899546921387, Loss: 2.1280462741851807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 92.15843236488756, Tokens per sec: 4204.520567478635, Loss: 2.120065212249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 91.70712865044348, Tokens per sec: 4183.93085364706, Loss: 2.1224215030670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 91.57307231225725, Tokens per sec: 4177.814835648042, Loss: 2.113819122314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 91.99542289804693, Tokens per sec: 4197.083628302939, Loss: 2.1409339904785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 92.15554503646928, Tokens per sec: 4204.388839633356, Loss: 2.1114704608917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 91.37405732516696, Tokens per sec: 4168.73522583931, Loss: 2.127502679824829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 92.15520456957597, Tokens per sec: 4204.373306599441, Loss: 2.1259520053863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 91.57055522232484, Tokens per sec: 4177.699999098424, Loss: 2.1378097534179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 91.36060016102417, Tokens per sec: 4168.12127308462, Loss: 2.119917154312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 92.01024521878902, Tokens per sec: 4197.759863247683, Loss: 2.131746768951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 91.95564154615472, Tokens per sec: 4195.268693869478, Loss: 2.1099917888641357 +------------------------------------------------------------------ + eval ppl=8.032726287841797, eval loss=2.083523988723755 +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 92.24157616238483, Tokens per sec: 4208.31381566727, Loss: 2.109541654586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 91.88211734633148, Tokens per sec: 4191.914318122935, Loss: 2.120681047439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 91.52527450502214, Tokens per sec: 4175.634168524662, Loss: 2.112650156021118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 91.82997207177884, Tokens per sec: 4189.535307611066, Loss: 2.1148712635040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 91.71375853878018, Tokens per sec: 4184.233327345412, Loss: 2.1285669803619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 92.13053884810962, Tokens per sec: 4203.247988703339, Loss: 2.130971670150757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 91.4199974071761, Tokens per sec: 4170.831138440279, Loss: 2.10583758354187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 92.44389635911568, Tokens per sec: 4217.544218209309, Loss: 2.1299734115600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 91.73021285974848, Tokens per sec: 4184.9840186186875, Loss: 2.1350038051605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 91.59478877053155, Tokens per sec: 4178.805600064537, Loss: 2.1166293621063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 92.13992254510592, Tokens per sec: 4203.6760987092, Loss: 2.1498286724090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 92.15248290344866, Tokens per sec: 4204.249136722454, Loss: 2.1112632751464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 91.55741274837499, Tokens per sec: 4177.100403373846, Loss: 2.1237993240356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 92.26215231292015, Tokens per sec: 4209.252556115705, Loss: 2.1247169971466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 92.050426494539, Tokens per sec: 4199.593043304928, Loss: 2.1281700134277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 91.39870368105821, Tokens per sec: 4169.859660224737, Loss: 2.1006839275360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 92.131761903832, Tokens per sec: 4203.303787861461, Loss: 2.122481107711792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 92.41987655737852, Tokens per sec: 4216.448368943658, Loss: 2.140033721923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 91.71994225251908, Tokens per sec: 4184.515445334308, Loss: 2.0970191955566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 92.2682659094004, Tokens per sec: 4209.531475162859, Loss: 2.1244003772735596 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/optimizer.pt +[rank0]:[2024-11-01 07:43:59,012] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007669509002880659, 'preprocessing_with_comm': 0.0006411609938368201, 'state_converting': 7.29523178999807, : 7.305150404994492}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500, took 30.99s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000350 + eval ppl=8.018506050109863, eval loss=2.081752061843872 +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 92.08425764406618, Tokens per sec: 4201.136513179125, Loss: 2.1062755584716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 90.6837983075872, Tokens per sec: 4137.243715384674, Loss: 2.1145389080047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 91.05633961244155, Tokens per sec: 4154.240071966497, Loss: 2.11421537399292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 91.789129993104, Tokens per sec: 4187.671979911143, Loss: 2.1167800426483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 90.53568039327585, Tokens per sec: 4130.486169697826, Loss: 2.128769874572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 91.4775855483045, Tokens per sec: 4173.458467460575, Loss: 2.10330867767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 91.35570560458817, Tokens per sec: 4167.897969989333, Loss: 2.1214137077331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 91.03205986935251, Tokens per sec: 4153.132363463093, Loss: 2.1328189373016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 91.02488807734647, Tokens per sec: 4152.805166632361, Loss: 2.11409068107605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 91.8109116310251, Tokens per sec: 4188.665718001978, Loss: 2.141643524169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 91.02173347852238, Tokens per sec: 4152.661245177778, Loss: 2.1331686973571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 90.99950771504929, Tokens per sec: 4151.647244860579, Loss: 2.1055848598480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 91.85220956591161, Tokens per sec: 4190.549843113147, Loss: 2.109116792678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 92.01559867673689, Tokens per sec: 4198.004102689168, Loss: 2.122164011001587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 92.15276039949039, Tokens per sec: 4204.261796853328, Loss: 2.117889404296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 91.94463484273557, Tokens per sec: 4194.766537857071, Loss: 2.121182680130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 91.62307818537683, Tokens per sec: 4180.096240796011, Loss: 2.1251673698425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 91.89567409406618, Tokens per sec: 4192.532814154345, Loss: 2.1419081687927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 92.14504767640727, Tokens per sec: 4203.909921262542, Loss: 2.117002487182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 91.45301432301882, Tokens per sec: 4172.337460739532, Loss: 2.065850019454956 +------------------------------------------------------------------ + eval ppl=8.316447257995605, eval loss=2.1182351112365723 +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 92.06473893962595, Tokens per sec: 4200.2460163231835, Loss: 2.120774745941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 91.91649617085879, Tokens per sec: 4193.48277443346, Loss: 2.1074092388153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 91.85202613640313, Tokens per sec: 4190.54147455564, Loss: 2.0953140258789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 91.76416021645018, Tokens per sec: 4186.53279018306, Loss: 2.1281826496124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 92.07674001159175, Tokens per sec: 4200.793538157238, Loss: 2.1366679668426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 92.01910418568131, Tokens per sec: 4198.164033626338, Loss: 2.1081597805023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 91.86532747811076, Tokens per sec: 4191.14831826326, Loss: 2.1239397525787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 91.56263718448923, Tokens per sec: 4177.3387564852455, Loss: 2.1327810287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 91.93206207100297, Tokens per sec: 4194.19293351091, Loss: 2.1346263885498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 92.30747799794666, Tokens per sec: 4211.320438240419, Loss: 2.120816469192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 91.73089822900796, Tokens per sec: 4185.01528704496, Loss: 2.114872932434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 91.5556185445401, Tokens per sec: 4177.018546871598, Loss: 2.111799716949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 92.33092479661023, Tokens per sec: 4212.390145533532, Loss: 2.1241025924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 92.11628150231493, Tokens per sec: 4202.597529465989, Loss: 2.1229374408721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 91.43544427787144, Tokens per sec: 4171.535867067656, Loss: 2.131155252456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 92.11444242742354, Tokens per sec: 4202.513625823017, Loss: 2.131014108657837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 92.25652795662381, Tokens per sec: 4208.995957548217, Loss: 2.1231627464294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 91.66158998016928, Tokens per sec: 4181.853254550914, Loss: 2.130136728286743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 90.986811853273, Tokens per sec: 4151.068024808865, Loss: 2.099025249481201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 91.7327471694032, Tokens per sec: 4185.099640779288, Loss: 2.1029248237609863 +------------------------------------------------------------------ + eval ppl=8.205326080322266, eval loss=2.104783535003662 +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 91.79072225309639, Tokens per sec: 4187.744623181162, Loss: 2.1237001419067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 91.99942010990875, Tokens per sec: 4197.265992076436, Loss: 2.1204540729522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 92.48412641767398, Tokens per sec: 4219.37962387213, Loss: 2.075648784637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 91.87581956525344, Tokens per sec: 4191.626996069025, Loss: 2.122770309448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 91.43725950448754, Tokens per sec: 4171.61868268687, Loss: 2.111910581588745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 92.00652241060654, Tokens per sec: 4197.590018523001, Loss: 2.118692636489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 92.16066048175028, Tokens per sec: 4204.622220283872, Loss: 2.130777597427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 92.12944658550302, Tokens per sec: 4203.198156686054, Loss: 2.115811586380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 92.4058998060028, Tokens per sec: 4215.810711193652, Loss: 2.0980424880981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 91.71522602959732, Tokens per sec: 4184.300278303289, Loss: 2.1307806968688965 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/optimizer.pt +[rank0]:[2024-11-01 08:36:47,887] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007848731998819858, 'preprocessing_with_comm': 0.0008058719977270812, 'state_converting': 7.508530600993254, : 7.5187868409993825}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550, took 31.76s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000400 +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 89.09172773422145, Tokens per sec: 4064.609087181687, Loss: 2.127333879470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 91.60417673124954, Tokens per sec: 4179.233904593082, Loss: 2.1024913787841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 92.42461613313466, Tokens per sec: 4216.664601395062, Loss: 2.1006100177764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 91.68342599321132, Tokens per sec: 4182.849473383972, Loss: 2.1244957447052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 91.44006838613663, Tokens per sec: 4171.746831575279, Loss: 2.1329822540283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 91.5713907087279, Tokens per sec: 4177.73811628071, Loss: 2.132084608078003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 91.65771998380326, Tokens per sec: 4181.676694697416, Loss: 2.1013333797454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 91.43137472101502, Tokens per sec: 4171.350202716991, Loss: 2.1185786724090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 91.58921844414066, Tokens per sec: 4178.551466489594, Loss: 2.102557897567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 91.41357691499532, Tokens per sec: 4170.538217969143, Loss: 2.1163856983184814 +------------------------------------------------------------------ + eval ppl=8.403345108032227, eval loss=2.1286299228668213 +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 91.93436526281268, Tokens per sec: 4194.298011441233, Loss: 2.0956146717071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 92.08494166928622, Tokens per sec: 4201.16772028663, Loss: 2.0964484214782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 92.72359951055482, Tokens per sec: 4230.305043484182, Loss: 2.0930445194244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 91.83174355450518, Tokens per sec: 4189.616127513995, Loss: 2.1184771060943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 91.77115046078272, Tokens per sec: 4186.8517043107595, Loss: 2.118987798690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 92.39474456783704, Tokens per sec: 4215.301778618526, Loss: 2.1043591499328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 91.77683296546996, Tokens per sec: 4187.110955767401, Loss: 2.1240828037261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 92.2130335052553, Tokens per sec: 4207.011621327891, Loss: 2.112387180328369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 92.86223583531435, Tokens per sec: 4236.630013038169, Loss: 2.1219077110290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 92.05088743664814, Tokens per sec: 4199.614072748773, Loss: 2.1043543815612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 91.41351292303592, Tokens per sec: 4170.535298480362, Loss: 2.1249873638153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 91.92433934526369, Tokens per sec: 4193.840601571504, Loss: 2.110307216644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 92.80862800886018, Tokens per sec: 4234.184276895306, Loss: 2.1214065551757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 91.59305753904714, Tokens per sec: 4178.726616533714, Loss: 2.0981574058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 92.09605173492129, Tokens per sec: 4201.674591966953, Loss: 2.104288101196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 92.14757761549667, Tokens per sec: 4204.02534402596, Loss: 2.117096424102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 91.95747947400149, Tokens per sec: 4195.352545181126, Loss: 2.109088182449341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 91.68427728623156, Tokens per sec: 4182.888311707506, Loss: 2.1071648597717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 92.50199541197654, Tokens per sec: 4220.194856424782, Loss: 2.115152597427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 91.51311216517355, Tokens per sec: 4175.079289207275, Loss: 2.1041994094848633 +------------------------------------------------------------------ + eval ppl=9.458279609680176, eval loss=2.2468905448913574 +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 91.46570407841665, Tokens per sec: 4172.916402201504, Loss: 2.0918285846710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 91.71972746640088, Tokens per sec: 4184.505646202222, Loss: 2.1050968170166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 91.84940575191393, Tokens per sec: 4190.421925424901, Loss: 2.1297547817230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 91.7960692990183, Tokens per sec: 4187.988570088428, Loss: 2.109898090362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 91.90104462748147, Tokens per sec: 4192.777832625508, Loss: 2.089630126953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 91.71994877893864, Tokens per sec: 4184.515743087477, Loss: 2.121478796005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 92.38211256684737, Tokens per sec: 4214.725472071133, Loss: 2.1140830516815186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 91.56983386966817, Tokens per sec: 4177.6670890108335, Loss: 2.120800733566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 91.61673075957543, Tokens per sec: 4179.806653813597, Loss: 2.096290349960327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 92.4247985060171, Tokens per sec: 4216.672921746391, Loss: 2.0968241691589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 91.75861148546822, Tokens per sec: 4186.27964184993, Loss: 2.1261954307556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 91.81366209630735, Tokens per sec: 4188.791201775439, Loss: 2.0821075439453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 92.0686854829122, Tokens per sec: 4200.426068457228, Loss: 2.0966596603393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 92.00459105195642, Tokens per sec: 4197.50190464173, Loss: 2.121424674987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 92.31906333783512, Tokens per sec: 4211.848993236335, Loss: 2.1014299392700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 91.98921179463748, Tokens per sec: 4196.800260722116, Loss: 2.1004462242126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 91.87736960736724, Tokens per sec: 4191.697713243577, Loss: 2.1218249797821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 92.44839889354697, Tokens per sec: 4217.749636185031, Loss: 2.1340157985687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 91.96321867739054, Tokens per sec: 4195.614383388124, Loss: 2.1054208278656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 91.60583763329099, Tokens per sec: 4179.309679501733, Loss: 2.1030631065368652 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/optimizer.pt +[rank0]:[2024-11-01 09:29:31,632] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007608895997691434, 'preprocessing_with_comm': 0.0006407190012396313, 'state_converting': 7.246919588003948, : 7.256745853999746}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000600, took 30.94s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000450 + eval ppl=8.721853256225586, eval loss=2.1658318042755127 +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 91.37456848330737, Tokens per sec: 4168.7585462762945, Loss: 2.0807526111602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 92.28226770794672, Tokens per sec: 4210.170275633517, Loss: 2.105698823928833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 92.03040214226387, Tokens per sec: 4198.679477407259, Loss: 2.102144479751587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 92.17717998438636, Tokens per sec: 4205.3758853236895, Loss: 2.080643892288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 91.69892552048724, Tokens per sec: 4183.556604349044, Loss: 2.1135292053222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 91.68857257785956, Tokens per sec: 4183.084274697867, Loss: 2.088813543319702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 92.20104087363094, Tokens per sec: 4206.464484565374, Loss: 2.118338108062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 92.09892697746761, Tokens per sec: 4201.80576842159, Loss: 2.110941171646118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 92.05822392020968, Tokens per sec: 4199.948783260182, Loss: 2.1110897064208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 91.67330919678301, Tokens per sec: 4182.387917370395, Loss: 2.1040358543395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 91.86842975630383, Tokens per sec: 4191.289852707087, Loss: 2.1046974658966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 91.78873794779334, Tokens per sec: 4187.654093728297, Loss: 2.1360044479370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 91.96926559726117, Tokens per sec: 4195.890260465388, Loss: 2.0806705951690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 91.63548198093382, Tokens per sec: 4180.662135985371, Loss: 2.128094434738159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 92.82216877885304, Tokens per sec: 4234.80204397829, Loss: 2.115901231765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 91.43917996262773, Tokens per sec: 4171.706299256967, Loss: 2.1082561016082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 91.54903922572515, Tokens per sec: 4176.718380293592, Loss: 2.1114630699157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 92.73710428806638, Tokens per sec: 4230.921168491405, Loss: 2.0955963134765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 91.88741371373654, Tokens per sec: 4192.1559529372, Loss: 2.1151702404022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 91.82322383594625, Tokens per sec: 4189.227434575205, Loss: 2.1121740341186523 +------------------------------------------------------------------ + eval ppl=8.601715087890625, eval loss=2.1519615650177 +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 91.94128800139659, Tokens per sec: 4194.613845771445, Loss: 2.1115779876708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 92.21117329918087, Tokens per sec: 4206.926753622365, Loss: 2.1058032512664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 92.18404739718282, Tokens per sec: 4205.689195539664, Loss: 2.118572950363159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 92.18890780889599, Tokens per sec: 4205.910940859002, Loss: 2.1012749671936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 92.648608987216, Tokens per sec: 4226.8837700352615, Loss: 2.081864833831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 92.10571850303788, Tokens per sec: 4202.115616454071, Loss: 2.116027593612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 91.87165937705605, Tokens per sec: 4191.437196867892, Loss: 2.1064348220825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 92.32415285323248, Tokens per sec: 4212.081190894407, Loss: 2.1006767749786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 91.91145788376005, Tokens per sec: 4193.252913950942, Loss: 2.095529794692993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 92.30184176254534, Tokens per sec: 4211.063297715566, Loss: 2.1110196113586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 92.4508017360948, Tokens per sec: 4217.859260455471, Loss: 2.113487958908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 91.73277663133695, Tokens per sec: 4185.10098491355, Loss: 2.0944013595581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 92.06187931022995, Tokens per sec: 4200.115552183307, Loss: 2.0852832794189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 91.97581844840015, Tokens per sec: 4196.189219515389, Loss: 2.1037065982818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 91.81790576659948, Tokens per sec: 4188.984809658808, Loss: 2.113269805908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 92.14706760008558, Tokens per sec: 4204.002075723423, Loss: 2.0794260501861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 92.3605707507552, Tokens per sec: 4213.7426753102045, Loss: 2.0936875343322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 92.1980331038779, Tokens per sec: 4206.327261855911, Loss: 2.076232671737671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 91.7921452233965, Tokens per sec: 4187.809542990887, Loss: 2.116295099258423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 92.39379607141733, Tokens per sec: 4215.258505609183, Loss: 2.110797882080078 +------------------------------------------------------------------ + eval ppl=8.307578086853027, eval loss=2.1171681880950928 +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 91.70833211997785, Tokens per sec: 4183.985759229541, Loss: 2.135772943496704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 92.15717853796822, Tokens per sec: 4204.46336445407, Loss: 2.11775541305542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 91.85645846987556, Tokens per sec: 4190.743689770988, Loss: 2.093872308731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 92.4889636914462, Tokens per sec: 4219.600313575112, Loss: 2.1205639839172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 92.32762620083072, Tokens per sec: 4212.239654542715, Loss: 2.094942092895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 91.69509051260987, Tokens per sec: 4183.381640766407, Loss: 2.114593029022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 92.5244280698175, Tokens per sec: 4221.218295830762, Loss: 2.1164305210113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 91.76963490944453, Tokens per sec: 4186.782560700035, Loss: 2.101205825805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 91.82833363914033, Tokens per sec: 4189.460557818233, Loss: 2.0929481983184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 92.26115400794686, Tokens per sec: 4209.207010703472, Loss: 2.106006383895874 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/model.pt +[rank0]:[2024-11-01 10:22:10,502] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007729671000561211, 'preprocessing_with_comm': 0.0006685019980068319, 'state_converting': 7.44814439000038, : 7.458099946998118}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000650, took 31.39s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000500 +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 91.10324128895546, Tokens per sec: 4156.379855147392, Loss: 2.081294536590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 91.08053449508574, Tokens per sec: 4155.34390890355, Loss: 2.107069730758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 91.64455130247678, Tokens per sec: 4181.075903320404, Loss: 2.126925468444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 92.32054760343509, Tokens per sec: 4211.916709505835, Loss: 2.1097593307495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 91.93906560714242, Tokens per sec: 4194.512454047314, Loss: 2.09106183052063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 92.40774312286385, Tokens per sec: 4215.894808366921, Loss: 2.097797393798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 92.07637114234497, Tokens per sec: 4200.776709330021, Loss: 2.107713222503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 91.81765506408725, Tokens per sec: 4188.973371922272, Loss: 2.088296890258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 92.30065654621181, Tokens per sec: 4211.009224894193, Loss: 2.114060401916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 91.5994993430222, Tokens per sec: 4179.020509307385, Loss: 2.115309953689575 +------------------------------------------------------------------ + eval ppl=8.564013481140137, eval loss=2.147568941116333 +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 92.00015963113438, Tokens per sec: 4197.299731063998, Loss: 2.136274576187134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 91.66369388613157, Tokens per sec: 4181.949240514037, Loss: 2.126535415649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 91.58047323543609, Tokens per sec: 4178.152486071615, Loss: 2.124274253845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 91.2148975874902, Tokens per sec: 4161.473921871738, Loss: 2.0917112827301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 92.19872223015922, Tokens per sec: 4206.35870168783, Loss: 2.0780067443847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 91.9732320757256, Tokens per sec: 4196.071222096963, Loss: 2.101767063140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 91.58380604053878, Tokens per sec: 4178.304537785625, Loss: 2.1020398139953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 92.19754847204177, Tokens per sec: 4206.3051516217165, Loss: 2.096489429473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 92.06775656025138, Tokens per sec: 4200.383688456474, Loss: 2.1133205890655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 91.99765375231962, Tokens per sec: 4197.18540599633, Loss: 2.1114277839660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 92.76069916103718, Tokens per sec: 4231.99763134073, Loss: 2.086590528488159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 91.96425830890719, Tokens per sec: 4195.661814230666, Loss: 2.086386203765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 91.5517442690188, Tokens per sec: 4176.841791791295, Loss: 2.1015079021453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 92.31601365858064, Tokens per sec: 4211.709858500419, Loss: 2.0959722995758057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 91.54383477730696, Tokens per sec: 4176.480939075734, Loss: 2.0860695838928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 91.71567911179335, Tokens per sec: 4184.32094915633, Loss: 2.111829996109009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 91.9689552986064, Tokens per sec: 4195.876103789296, Loss: 2.0937273502349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 91.9041962062776, Tokens per sec: 4192.921616298128, Loss: 2.0992822647094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 91.91233485540394, Tokens per sec: 4193.292923803748, Loss: 2.1203248500823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 91.50998007542778, Tokens per sec: 4174.93639467861, Loss: 2.0762174129486084 +------------------------------------------------------------------ + eval ppl=7.7750043869018555, eval loss=2.0509140491485596 +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 91.42578753324183, Tokens per sec: 4171.095299879519, Loss: 2.0917482376098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 91.2187249671058, Tokens per sec: 4161.648537432141, Loss: 2.1058592796325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 91.81583236858975, Tokens per sec: 4188.8902155521955, Loss: 2.1115524768829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 91.7060361979559, Tokens per sec: 4183.881012966884, Loss: 2.1073293685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 92.44792036363697, Tokens per sec: 4217.72780433745, Loss: 2.0851004123687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 92.35141080201608, Tokens per sec: 4213.324773313802, Loss: 2.0894336700439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 92.3024909471855, Tokens per sec: 4211.092915300214, Loss: 2.0964128971099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 92.50591201311813, Tokens per sec: 4220.373542516027, Loss: 2.1002230644226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 91.76726255587522, Tokens per sec: 4186.674327420457, Loss: 2.0918331146240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 91.73778196181311, Tokens per sec: 4185.329341824513, Loss: 2.089984893798828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 92.18353460366869, Tokens per sec: 4205.665800492443, Loss: 2.106719493865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 92.08686802244792, Tokens per sec: 4201.255605803786, Loss: 2.112452268600464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 91.82600896924887, Tokens per sec: 4189.354499998884, Loss: 2.1064491271972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 91.71257963019258, Tokens per sec: 4184.179542300717, Loss: 2.0951712131500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 92.35851233088833, Tokens per sec: 4213.648764547569, Loss: 2.1063880920410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 92.1044128966333, Tokens per sec: 4202.056051107306, Loss: 2.114764928817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 92.12272098449694, Tokens per sec: 4202.891316313088, Loss: 2.101835250854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 91.75510495529443, Tokens per sec: 4186.119664321473, Loss: 2.1033313274383545 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/model.pt +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/model.pt +[rank0]:[2024-11-01 11:12:48,296] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007689733000006527, 'preprocessing_with_comm': 0.000686487001075875, 'state_converting': 7.311131950002164, : 7.321103104994108}) +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000698, took 31.40s +Deleting checkpoint /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000550 \ No newline at end of file diff --git a/wandb/run-20241031_225437-1g18vgup/files/requirements.txt b/wandb/run-20241031_225437-1g18vgup/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20241031_225437-1g18vgup/files/wandb-metadata.json b/wandb/run-20241031_225437-1g18vgup/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4929e9ffbd050b36bdf125c565a748dca4c7b898 --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-10-31T13:54:37.885597", + "startedAt": "2024-10-31T13:54:37.309116", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "1", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "256", + "--train-iters", + "698", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "--train-data-path", + "183190553", + "/project/trans/datasets/train/documents_text_document", + "--valid-data-path", + "61085876", + "/project/trans/datasets/valid/documents_text_document", + "--test-data-path", + "61085876", + "/project/trans/datasets/valid/documents_text_document", + "--lr", + "1.5e-4", + "--min-lr", + "1.5e-5", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "70", + "--lr-decay-iters", + "698", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "50", + "--eval-interval", + "20", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "--save", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs", + "--load", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "3", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "trans_experiment", + "--wandb-name", + "llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-22:54:20" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0319999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json b/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..53f8c9e75ac7ab811883d2f7b645d53bc58e2d35 --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.1033313274383545, "training/perplexity": 8.193419461848551, "utils/batch_size": 1, "utils/global_batch_size": 256, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 256, "utils/iteration": 698, "optimizer/lr": 1.5e-05, "optimizer/variance_l2": 0.003787014461590841, "optimizer/variance_sqrt_l2": 0.48841666823350643, "optimizer/momentum_l2": 0.10994280162474024, "optimizer/weight_l2": 1150.0504336767149, "optimizer/variance_l1": 0.2378997802734375, "optimizer/variance_sqrt_l1": 12692.0, "optimizer/momentum_l1": 2361.0, "optimizer/weight_l1": 52150272.0, "optimizer/variance_abs_max": 0.0021514892578125, "optimizer/variance_sqrt_abs_max": 0.04638671875, "optimizer/momentum_abs_max": 0.0081787109375, "optimizer/weight_abs_max": 6.5, "stats/1_iteration_time": 62.68334902999777, "stats/tokens_per_sec": 4186.119664321473, "stats/tokens_per_sec_per_gpu": 4186.119664321473, "stats/tflops": 91.75510495529443, "_timestamp": 1730427150.9565277, "_runtime": 44273.63433265686, "_step": 698, "evaluation/val_loss": 2.0509140491485596, "evaluation/val_ppl": 7.7750043869018555, "_wandb": {"runtime": 44306}} \ No newline at end of file diff --git a/wandb/run-20241031_225437-1g18vgup/logs/debug-internal.log b/wandb/run-20241031_225437-1g18vgup/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..25567e717daab90c6174557323d709bc5b6f2aac --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/logs/debug-internal.log @@ -0,0 +1,23388 @@ +2024-10-31 22:54:37,324 INFO StreamThr :31085 [internal.py:wandb_internal():86] W&B internal server running at pid: 31085, started at: 2024-10-31 22:54:37.323596 +2024-10-31 22:54:37,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status +2024-10-31 22:54:37,327 INFO WriterThread:31085 [datastore.py:open_for_write():87] open: /project/wandb/run-20241031_225437-1g18vgup/run-1g18vgup.wandb +2024-10-31 22:54:37,328 DEBUG SenderThread:31085 [sender.py:send():382] send: header +2024-10-31 22:54:37,342 DEBUG SenderThread:31085 [sender.py:send():382] send: run +2024-10-31 22:54:37,785 INFO SenderThread:31085 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20241031_225437-1g18vgup/files +2024-10-31 22:54:37,785 INFO SenderThread:31085 [sender.py:_start_run_threads():1136] run started: 1g18vgup with start time 1730382877.322195 +2024-10-31 22:54:37,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: check_version +2024-10-31 22:54:37,790 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: check_version +2024-10-31 22:54:37,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: run_start +2024-10-31 22:54:37,873 DEBUG HandlerThread:31085 [system_info.py:__init__():27] System info init +2024-10-31 22:54:37,873 DEBUG HandlerThread:31085 [system_info.py:__init__():42] System info init done +2024-10-31 22:54:37,873 INFO HandlerThread:31085 [system_monitor.py:start():194] Starting system monitor +2024-10-31 22:54:37,873 INFO SystemMonitor:31085 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-10-31 22:54:37,873 INFO HandlerThread:31085 [system_monitor.py:probe():214] Collecting system info +2024-10-31 22:54:37,873 INFO SystemMonitor:31085 [interfaces.py:start():190] Started cpu monitoring +2024-10-31 22:54:37,874 INFO SystemMonitor:31085 [interfaces.py:start():190] Started disk monitoring +2024-10-31 22:54:37,875 INFO SystemMonitor:31085 [interfaces.py:start():190] Started gpu monitoring +2024-10-31 22:54:37,876 INFO SystemMonitor:31085 [interfaces.py:start():190] Started memory monitoring +2024-10-31 22:54:37,876 INFO SystemMonitor:31085 [interfaces.py:start():190] Started network monitoring +2024-10-31 22:54:37,885 DEBUG HandlerThread:31085 [system_info.py:probe():151] Probing system +2024-10-31 22:54:37,887 DEBUG HandlerThread:31085 [system_info.py:_probe_git():136] Probing git +2024-10-31 22:54:37,899 DEBUG HandlerThread:31085 [system_info.py:_probe_git():144] Probing git done +2024-10-31 22:54:37,899 DEBUG HandlerThread:31085 [system_info.py:probe():199] Probing system done +2024-10-31 22:54:37,899 DEBUG HandlerThread:31085 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-10-31T13:54:37.885597', 'startedAt': '2024-10-31T13:54:37.309116', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '1', '--valid_micro_batch_size', '1', '--global-batch-size', '256', '--train-iters', '698', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', '--train-data-path', '183190553', '/project/trans/datasets/train/documents_text_document', '--valid-data-path', '61085876', '/project/trans/datasets/valid/documents_text_document', '--test-data-path', '61085876', '/project/trans/datasets/valid/documents_text_document', '--lr', '1.5e-4', '--min-lr', '1.5e-5', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '70', '--lr-decay-iters', '698', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '50', '--eval-interval', '20', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', '--save', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', '--load', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'NO_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '3', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'trans_experiment', '--wandb-name', 'llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-22:54:20'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0319999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-10-31 22:54:37,899 INFO HandlerThread:31085 [system_monitor.py:probe():224] Finished collecting system info +2024-10-31 22:54:37,899 INFO HandlerThread:31085 [system_monitor.py:probe():227] Publishing system info +2024-10-31 22:54:37,901 INFO HandlerThread:31085 [system_monitor.py:probe():229] Finished publishing system info +2024-10-31 22:54:37,906 DEBUG SenderThread:31085 [sender.py:send():382] send: files +2024-10-31 22:54:37,906 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-10-31 22:54:37,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: python_packages +2024-10-31 22:54:37,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:54:37,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:54:37,920 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: python_packages +2024-10-31 22:54:37,922 DEBUG SenderThread:31085 [sender.py:send():382] send: telemetry +2024-10-31 22:54:37,922 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:54:38,580 INFO wandb-upload_0:31085 [upload_job.py:push():131] Uploaded file /tmp/tmpw6rfh0mwwandb/w56ov3v3-wandb-metadata.json +2024-10-31 22:54:38,786 INFO Thread-12 :31085 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_225437-1g18vgup/files/requirements.txt +2024-10-31 22:54:38,787 INFO Thread-12 :31085 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-metadata.json +2024-10-31 22:54:38,787 INFO Thread-12 :31085 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:54:40,787 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:54:42,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:54:47,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:54:52,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:54:52,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:54:52,917 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:54:52,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:54:58,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:03,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:07,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:55:07,917 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:55:07,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:55:08,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:08,806 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/config.yaml +2024-10-31 22:55:13,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:18,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:22,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:55:22,917 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:55:22,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:55:24,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:29,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:34,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:37,877 DEBUG SystemMonitor:31085 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-10-31 22:55:37,878 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:55:37,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:55:37,917 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:55:37,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:55:40,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:45,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:50,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:55:52,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:55:52,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:55:52,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:55:56,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:01,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:02,837 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:56:06,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:07,879 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:56:07,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:56:07,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:56:07,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:56:11,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:16,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:21,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:22,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:56:22,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:56:22,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:56:27,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:32,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:37,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:37,880 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:56:37,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:56:37,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:56:37,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:56:42,862 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:56:42,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:44,778 DEBUG SenderThread:31085 [sender.py:send():382] send: config +2024-10-31 22:56:44,778 DEBUG SenderThread:31085 [sender.py:send():382] send: config +2024-10-31 22:56:44,863 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:56:46,864 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:56:48,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:52,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:56:52,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:56:52,920 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:56:54,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:56:59,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:04,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:07,881 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:57:07,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:57:07,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:57:07,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:57:09,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:14,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:14,879 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/config.yaml +2024-10-31 22:57:19,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:22,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:57:22,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:57:22,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:57:25,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:30,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:35,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:37,882 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:57:37,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:57:37,918 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:57:37,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:57:40,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:45,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:50,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:57:53,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:57:53,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:57:54,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:57:55,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:00,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:02,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 22:58:04,905 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:58:05,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:07,883 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:58:08,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:58:08,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:58:08,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:58:11,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:16,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:21,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:23,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:58:23,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:58:24,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:58:26,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:31,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:36,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:37,884 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:58:38,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:58:38,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:58:39,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:58:42,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:47,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:52,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:58:53,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:58:53,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:58:54,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:58:57,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:02,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:05,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 22:59:05,494 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 22:59:05,495 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 22:59:05,497 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 22:59:05,938 INFO Thread-12 :31085 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 22:59:06,939 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 22:59:07,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:07,885 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:59:08,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:59:08,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:59:08,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:59:13,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:18,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:23,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:23,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:59:23,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:59:24,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:59:29,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:34,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:37,886 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 22:59:38,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:59:38,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:59:39,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:59:40,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:45,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:50,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 22:59:53,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 22:59:53,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 22:59:54,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 22:59:55,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:00,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:05,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:07,888 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:00:07,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:00:07,911 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:00:07,911 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:00:07,912 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:00:07,971 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:00:08,971 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:00:08,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:00:08,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:00:08,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:00:11,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:16,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:21,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:23,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:00:23,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:00:23,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:00:27,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:32,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:37,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:37,920 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:00:38,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:00:38,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:00:39,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:00:42,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:47,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:52,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:00:53,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:00:53,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:00:54,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:00:58,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:03,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:07,892 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:01:08,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:08,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:01:08,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:01:09,046 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:01:10,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:01:10,678 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:01:10,678 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:01:10,680 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:01:11,020 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:01:13,022 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:01:14,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:19,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:23,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:01:23,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:01:24,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:01:25,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:30,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:35,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:37,893 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:01:38,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:01:38,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:01:39,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:01:40,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:45,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:50,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:01:53,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:01:53,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:01:54,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:01:56,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:01,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:06,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:07,894 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:02:08,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:02:08,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:02:09,046 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:02:12,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:13,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:02:13,481 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:02:13,481 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:02:13,483 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:02:14,054 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:02:15,054 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:02:17,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:22,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:23,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:02:23,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:02:24,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:02:28,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:33,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:37,894 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:02:38,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:38,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:02:38,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:02:39,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:02:44,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:49,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:53,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:02:53,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:02:54,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:02:54,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:02:59,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:04,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:07,895 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:03:08,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:03:08,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:03:09,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:03:09,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:14,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:16,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:03:16,422 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:03:16,422 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:03:16,424 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:03:17,088 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:03:19,090 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:03:19,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:23,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:03:23,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:03:24,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:03:25,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:30,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:35,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:37,896 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:03:38,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:03:38,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:03:39,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:03:40,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:45,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:50,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:03:53,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:03:53,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:03:54,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:03:55,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:00,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:05,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:07,897 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:04:08,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:04:08,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:04:09,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:04:10,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:15,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:18,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:04:18,538 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:04:18,538 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:04:18,539 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:04:19,122 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:04:20,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:21,123 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:04:23,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:04:23,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:04:24,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:04:26,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:31,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:36,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:37,898 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:04:38,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:04:38,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:04:39,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:04:42,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:47,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:52,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:04:53,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:04:53,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:04:54,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:04:57,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:02,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:07,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:07,899 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:05:08,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:05:09,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:05:09,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:05:12,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:17,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:21,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:05:21,366 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:05:21,367 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:05:21,368 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:05:22,156 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:05:22,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:23,157 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:05:23,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:05:24,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:05:24,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:05:28,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:33,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:37,900 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:05:38,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:39,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:05:39,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:05:39,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:05:44,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:49,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:54,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:05:54,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:05:54,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:05:54,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:05:59,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:04,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:07,901 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:06:09,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:06:09,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:06:09,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:06:10,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:15,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:20,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:24,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:06:24,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:06:24,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:06:24,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:06:24,419 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:06:24,419 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:06:24,420 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:06:25,191 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:06:25,192 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:06:25,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:30,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:35,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:37,902 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:06:39,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:06:39,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:06:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:06:41,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:46,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:51,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:06:54,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:06:54,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:06:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:06:56,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:01,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:06,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:07,903 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:07:09,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:07:09,000 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:07:09,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:07:12,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:17,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:22,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:24,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:07:24,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:07:24,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:07:26,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:07:26,831 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:07:26,831 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:07:26,833 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:07:27,230 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:07:27,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:29,231 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:07:32,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:37,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:37,904 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:07:39,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:07:39,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:07:39,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:07:43,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:48,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:53,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:07:54,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:07:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:07:54,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:07:59,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:04,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:07,908 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:08:09,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:08:09,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:08:09,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:08:10,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:15,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:20,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:24,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:08:24,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:08:24,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:08:25,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:29,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:08:29,588 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:08:29,588 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:08:29,589 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:08:30,265 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:08:30,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:31,266 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:08:35,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:37,907 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:08:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:08:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:08:39,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:08:41,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:46,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:51,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:08:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:08:54,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:08:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:08:57,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:02,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:07,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:07,908 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:09:09,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:09:09,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:09:09,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:09:13,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:18,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:23,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:24,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:09:24,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:09:24,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:09:29,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:32,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:09:32,149 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:09:32,150 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:09:32,151 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:09:32,300 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:09:33,301 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:09:35,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:37,909 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:09:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:09:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:09:39,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:09:40,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:45,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:50,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:09:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:09:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:09:54,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:09:56,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:01,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:06,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:07,910 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:10:09,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:10:09,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:10:09,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:10:12,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:17,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:22,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:24,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:10:24,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:10:24,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:10:27,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:32,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:34,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:10:34,709 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:10:34,710 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:10:34,712 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:10:35,335 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:10:35,335 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:10:37,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:37,910 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:10:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:10:39,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:10:39,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:10:43,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:48,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:53,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:10:54,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:10:54,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:10:54,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:10:59,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:04,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:07,911 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:11:09,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:11:09,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:11:09,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:11:10,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:15,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:20,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:24,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:11:24,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:11:24,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:11:25,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:30,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:35,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:37,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:11:37,536 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:11:37,536 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:11:37,538 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:11:37,912 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:11:38,370 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:11:39,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:11:39,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:11:39,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:11:39,371 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:11:41,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:46,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:51,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:11:54,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:11:54,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:11:54,004 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:11:57,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:02,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:07,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:07,913 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:12:09,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:12:09,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:12:09,004 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:12:12,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:17,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:22,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:24,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:12:24,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:12:24,004 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:12:28,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:33,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:37,914 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:12:38,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:39,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:12:39,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:12:39,004 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:12:40,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:12:40,140 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:12:40,141 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:12:40,142 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:12:40,406 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:12:41,406 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:12:44,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:49,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:54,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:12:54,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:12:54,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:12:54,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:12:59,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:04,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:07,938 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:13:09,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:13:09,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:13:09,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:13:09,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:14,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:19,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:24,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:13:24,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:13:24,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:13:25,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:30,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:35,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:37,939 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:13:39,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:13:39,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:13:39,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:13:40,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:42,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:13:42,895 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:13:42,896 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:13:42,897 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:13:43,440 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:13:43,440 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:13:45,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:50,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:13:54,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:13:54,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:13:54,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:13:56,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:01,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:06,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:07,940 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:14:09,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:14:09,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:14:09,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:14:12,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:17,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:22,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:24,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:14:24,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:14:24,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:14:27,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:32,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:37,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:37,941 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:14:39,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:14:39,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:14:39,005 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:14:42,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:45,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:14:45,866 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:14:45,866 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:14:45,867 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:14:46,474 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:14:47,475 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:14:47,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:52,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:14:54,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:14:54,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:14:54,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:14:58,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:03,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:07,942 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:15:08,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:09,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:15:09,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:15:09,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:15:14,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:19,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:24,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:15:24,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:15:24,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:15:25,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:30,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:35,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:37,943 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:15:39,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:15:39,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:15:39,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:15:40,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:45,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:48,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:15:48,952 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:15:48,953 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:15:48,954 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:15:49,508 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:15:50,509 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:15:50,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:15:54,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:15:54,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:15:54,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:15:56,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:01,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:06,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:07,944 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:16:09,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:16:09,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:16:09,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:16:12,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:17,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:22,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:24,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:16:24,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:16:24,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:16:27,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:32,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:37,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:37,945 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:16:39,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:16:39,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:16:39,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:16:43,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:48,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:51,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:16:51,659 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:16:51,659 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:16:51,661 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:16:52,543 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:16:53,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:16:54,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:16:54,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:16:54,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:16:54,544 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:16:59,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:04,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:07,946 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:17:09,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:17:09,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:17:09,006 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:17:10,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:15,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:20,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:24,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:17:24,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:17:24,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:17:25,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:30,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:35,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:37,947 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:17:39,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:17:39,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:17:39,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:17:41,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:46,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:51,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:54,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:17:54,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:17:54,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:17:54,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:17:54,253 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:17:54,253 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:17:54,254 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:17:54,577 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:17:55,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:17:56,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:17:56,578 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:17:58,579 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:18:01,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:06,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:07,948 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:18:09,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:18:09,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:18:09,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:18:12,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:17,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:22,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:24,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:18:24,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:18:24,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:18:27,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:32,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:37,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:37,948 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:18:39,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:18:39,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:18:39,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:18:43,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:48,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:53,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:54,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:18:54,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:18:54,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:18:58,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:18:58,477 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:18:58,478 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:18:58,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:18:58,480 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:18:58,611 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:19:00,613 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:19:03,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:07,949 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:19:08,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:09,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:19:09,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:19:09,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:19:14,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:19,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:24,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:19:24,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:19:24,007 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:19:24,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:29,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:34,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:37,950 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:19:39,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:19:39,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:19:39,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:19:40,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:45,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:50,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:19:54,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:19:54,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:19:54,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:19:55,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:00,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:00,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:20:00,853 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:20:00,854 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:20:00,855 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:20:01,653 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:20:02,662 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:20:05,856 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:07,951 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:20:09,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:20:09,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:20:09,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:20:11,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:16,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:21,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:24,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:20:24,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:20:24,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:20:26,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:31,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:36,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:37,952 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:20:39,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:20:39,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:20:39,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:20:42,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:47,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:52,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:20:54,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:20:54,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:20:54,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:20:58,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:03,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:03,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:21:03,350 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:21:03,351 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:21:03,352 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:21:03,695 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:21:04,695 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:21:07,953 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:21:08,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:09,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:21:09,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:21:09,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:21:14,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:19,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:24,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:21:24,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:21:24,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:21:24,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:29,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:34,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:37,954 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:21:39,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:21:39,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:21:39,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:21:40,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:45,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:50,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:21:54,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:21:54,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:21:54,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:21:55,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:00,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:05,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:05,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:22:05,598 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:22:05,598 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:22:05,600 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:22:05,728 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:22:07,955 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:22:08,730 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:22:09,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:22:09,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:22:09,008 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:22:11,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:16,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:21,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:24,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:22:24,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:22:24,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:22:28,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:33,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:37,955 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:22:38,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:39,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:22:39,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:22:39,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:22:44,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:49,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:22:54,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:22:54,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:22:54,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:22:55,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:00,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:05,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:07,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:23:07,489 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:23:07,489 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:23:07,490 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:23:07,763 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:23:07,956 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:23:08,763 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:23:09,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:23:09,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:23:09,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:23:10,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:15,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:20,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:24,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:23:24,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:23:24,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:23:26,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:31,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:36,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:37,959 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:23:39,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:23:39,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:23:39,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:23:42,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:47,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:52,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:23:54,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:23:54,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:23:54,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:23:57,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:02,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:07,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:07,958 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:24:09,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:24:09,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:24:09,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:24:09,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:24:09,931 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:24:09,931 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:24:09,933 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:24:10,798 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:24:12,799 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:24:12,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:17,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:22,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:24,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:24:24,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:24:24,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:24:28,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:33,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:37,959 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:24:38,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:39,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:24:39,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:24:39,009 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:24:44,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:49,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:54,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:24:54,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:24:54,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:24:54,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:24:59,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:04,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:07,960 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:25:09,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:25:09,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:25:09,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:25:10,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:12,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:25:12,407 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:25:12,407 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:25:12,408 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:25:12,833 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:25:14,834 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:25:15,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:20,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:24,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:25:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:25:24,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:25:26,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:31,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:36,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:37,961 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:25:39,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:25:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:25:39,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:25:42,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:47,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:52,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:25:54,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:25:54,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:25:54,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:25:58,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:03,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:07,962 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:26:08,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:09,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:26:09,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:26:09,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:26:14,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:14,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:26:14,915 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:26:14,916 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:26:14,917 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:26:15,874 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:26:16,874 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:26:19,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:24,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:26:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:26:24,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:26:25,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:30,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:35,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:37,963 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:26:39,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:26:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:26:39,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:26:40,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:45,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:50,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:26:54,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:26:54,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:26:54,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:26:56,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:01,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:06,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:07,964 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:27:09,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:27:09,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:27:09,010 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:27:11,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:16,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:17,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:27:17,660 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:27:17,661 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:27:17,662 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:27:17,910 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:27:18,911 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:27:21,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:24,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:27:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:27:24,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:27:27,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:32,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:37,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:37,965 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:27:39,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:27:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:27:39,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:27:42,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:47,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:52,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:27:54,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:27:54,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:27:54,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:27:58,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:03,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:07,966 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:28:08,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:09,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:28:09,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:28:09,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:28:14,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:19,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:20,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:28:20,156 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:28:20,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:28:20,158 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:28:20,960 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:28:22,961 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:28:24,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:28:24,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:28:24,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:28:25,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:30,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:35,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:37,967 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:28:39,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:28:39,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:28:39,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:28:41,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:46,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:51,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:28:54,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:28:54,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:28:54,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:28:56,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:01,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:06,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:07,968 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:29:09,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:29:09,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:29:09,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:29:12,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:17,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:22,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:22,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:29:22,798 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:29:22,798 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:29:22,800 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:29:23,003 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:29:24,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:29:24,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:29:24,011 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:29:25,004 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:29:27,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:32,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:37,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:37,969 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:29:39,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:29:39,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:29:39,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:29:43,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:48,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:53,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:29:54,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:29:54,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:29:54,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:29:58,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:03,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:07,970 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:30:08,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:09,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:30:09,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:30:09,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:30:14,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:19,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:30:24,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:30:24,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:30:25,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:30:25,125 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:30:25,126 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:30:25,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:25,134 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:30:26,038 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:30:27,039 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:30:30,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:35,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:37,970 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:30:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:30:39,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:30:39,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:30:40,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:45,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:50,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:30:54,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:30:54,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:30:54,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:30:55,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:00,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:05,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:07,973 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:31:09,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:31:09,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:31:09,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:31:10,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:15,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:20,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:31:24,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:31:24,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:31:26,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:27,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:31:27,438 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:31:27,438 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:31:27,440 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:31:28,073 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:31:29,073 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:31:31,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:36,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:37,972 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:31:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:31:39,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:31:39,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:31:42,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:47,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:52,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:31:54,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:31:54,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:31:54,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:31:57,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:02,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:07,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:07,974 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:32:09,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:32:09,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:32:09,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:32:13,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:18,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:23,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:32:24,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:32:24,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:32:29,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:29,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:32:29,771 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:32:29,771 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:32:29,773 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:32:30,109 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:32:31,110 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:32:34,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:37,974 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:32:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:32:39,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:32:39,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:32:40,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:45,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:50,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:32:54,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:32:54,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:32:54,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:32:55,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:00,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:05,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:07,975 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:33:09,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:33:09,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:33:09,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:33:11,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:16,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:21,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:24,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:33:24,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:33:24,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:33:27,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:32,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:33:32,143 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:33:32,143 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:33:32,144 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:33:33,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:33,145 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:33:33,145 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:33:37,976 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:33:38,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:39,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:33:39,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:33:39,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:33:44,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:49,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:33:54,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:33:54,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:33:54,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:33:55,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:00,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:05,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:07,977 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:34:09,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:34:09,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:34:09,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:34:11,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:16,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:21,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:24,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:34:24,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:34:24,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:34:26,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:31,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:34,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:34:34,804 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:34:34,804 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:34:34,805 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:34:35,179 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:34:36,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:37,181 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:34:37,977 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:34:39,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:34:39,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:34:39,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:34:42,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:47,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:52,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:34:54,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:34:54,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:34:54,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:34:57,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:02,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:07,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:07,978 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:35:09,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:35:09,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:35:09,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:35:13,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:18,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:23,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:24,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:35:24,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:35:24,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:35:28,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:33,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:37,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:35:37,343 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:35:37,343 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:35:37,344 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:35:37,979 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:35:38,215 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:35:38,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:39,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:35:39,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:35:39,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:35:39,215 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:35:44,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:49,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:54,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:35:54,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:35:54,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:35:54,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:35:59,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:04,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:07,980 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:36:09,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:36:09,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:36:09,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:36:10,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:15,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:20,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:24,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:36:24,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:36:24,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:36:25,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:30,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:35,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:37,981 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:36:39,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:36:39,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:36:39,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:36:39,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:36:39,807 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:36:39,808 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:36:39,809 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:36:40,250 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:36:40,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:41,250 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:36:45,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:50,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:36:54,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:36:54,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:36:54,014 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:36:56,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:01,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:06,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:07,982 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:37:09,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:37:09,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:37:09,015 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:37:11,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:16,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:21,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:24,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:37:24,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:37:24,015 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:37:26,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:31,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:36,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:37,983 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:37:39,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:37:39,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:37:39,015 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:37:42,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:42,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:37:42,416 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:37:42,417 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:37:42,418 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:37:43,285 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:37:43,285 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:37:47,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:52,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:37:54,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:37:54,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:37:54,015 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:37:58,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:03,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:07,984 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:38:08,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:09,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:38:09,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:38:09,015 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:38:14,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:19,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:24,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:38:24,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:38:24,018 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:38:25,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:30,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:35,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:37,985 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:38:39,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:38:39,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:38:39,018 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:38:40,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:44,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:38:44,880 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:38:44,880 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:38:44,882 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:38:45,320 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:38:45,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:45,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:38:47,322 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:38:50,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:38:54,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:38:54,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:38:54,018 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:38:56,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:01,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:06,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:07,985 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:39:09,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:39:09,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:39:09,018 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:39:11,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:16,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:21,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:24,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:39:24,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:39:24,018 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:39:27,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:32,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:37,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:37,986 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:39:39,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:39:39,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:39:39,018 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:39:42,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:47,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:47,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:39:47,961 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:39:47,961 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:39:47,962 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:39:48,356 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:39:49,356 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:39:52,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:39:54,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:39:54,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:39:54,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:39:58,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:03,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:07,987 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:40:08,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:09,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:40:09,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:40:09,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:40:14,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:19,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:24,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:40:24,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:40:24,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:40:25,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:30,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:35,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:37,988 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:40:39,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:40:39,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:40:39,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:40:40,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:45,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:50,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:40:50,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:40:50,870 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:40:50,870 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:40:50,871 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:40:51,392 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:40:53,394 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:40:54,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:40:54,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:40:54,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:40:56,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:01,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:06,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:07,989 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:41:09,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:41:09,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:41:09,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:41:11,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:16,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:21,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:24,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:41:24,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:41:24,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:41:27,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:32,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:37,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:37,990 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:41:39,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:41:39,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:41:39,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:41:43,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:48,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:53,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:41:53,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:41:53,207 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:41:53,207 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:41:53,208 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:41:53,427 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:41:54,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:41:54,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:41:54,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:41:55,429 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:41:58,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:03,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:07,991 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:42:08,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:09,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:42:09,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:42:09,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:42:14,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:19,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:24,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:42:24,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:42:24,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:42:24,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:29,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:34,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:37,992 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:42:39,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:42:39,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:42:39,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:42:39,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:44,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:49,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:54,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:42:54,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:42:54,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:42:55,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:42:55,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:42:55,696 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:42:55,696 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:42:55,697 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:42:56,464 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:42:57,465 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:43:00,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:05,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:07,992 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:43:09,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:43:09,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:43:09,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:43:11,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:16,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:21,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:24,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:43:24,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:43:24,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:43:26,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:31,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:36,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:37,993 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:43:39,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:43:39,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:43:39,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:43:41,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:46,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:51,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:54,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:43:54,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:43:54,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:43:57,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:43:58,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:43:58,518 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:43:58,518 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:43:58,519 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:43:59,500 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:43:59,500 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:44:02,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:07,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:07,994 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:44:09,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:44:09,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:44:09,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:44:13,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:18,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:23,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:24,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:44:24,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:44:24,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:44:29,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:34,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:37,995 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:44:39,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:44:39,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:44:39,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:44:39,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:44,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:49,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:54,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:44:54,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:44:54,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:44:54,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:44:59,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:00,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:45:00,946 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:45:00,947 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:45:00,948 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:45:01,534 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:45:03,535 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:45:04,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:07,996 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:45:09,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:45:09,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:45:09,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:45:10,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:15,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:20,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:24,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:45:24,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:45:24,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:45:25,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:30,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:35,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:37,997 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:45:39,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:45:39,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:45:39,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:45:40,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:45,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:50,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:45:54,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:45:54,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:45:54,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:45:56,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:01,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:03,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:46:03,650 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:46:03,650 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:46:03,651 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:46:04,569 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:46:05,582 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:46:06,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:07,998 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:46:09,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:46:09,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:46:09,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:46:12,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:17,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:22,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:24,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:46:24,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:46:24,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:46:27,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:32,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:37,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:38,001 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:46:39,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:46:39,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:46:39,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:46:42,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:47,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:52,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:46:54,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:46:54,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:46:54,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:46:57,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:02,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:06,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:47:06,081 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:47:06,081 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:47:06,082 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:47:06,616 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:47:07,616 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:47:08,000 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:47:08,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:09,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:47:09,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:47:09,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:47:13,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:18,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:23,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:24,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:47:24,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:47:24,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:47:29,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:34,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:38,002 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:47:39,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:47:39,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:47:39,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:47:39,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:44,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:49,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:47:54,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:47:54,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:47:54,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:47:55,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:00,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:05,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:08,001 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:48:08,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:48:08,827 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:48:08,828 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:48:08,829 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:48:09,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:48:09,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:48:09,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:48:09,652 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:48:09,652 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:48:11,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:16,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:21,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:24,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:48:24,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:48:24,022 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:48:26,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:31,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:36,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:38,002 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:48:39,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:48:39,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:48:39,022 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:48:42,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:47,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:52,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:48:54,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:48:54,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:48:54,022 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:48:58,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:03,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:08,003 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:49:09,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:09,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:49:09,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:49:09,022 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:49:11,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:49:11,521 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:49:11,522 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:49:11,523 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:49:11,687 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:49:13,688 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:49:14,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:19,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:23,694 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:49:24,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:49:24,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:49:24,022 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:49:25,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:29,698 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:49:30,699 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:49:31,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:36,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:38,004 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:49:39,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:49:39,184 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:49:39,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:49:41,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:43,712 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:49:46,714 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:49:47,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:52,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:49:54,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:49:54,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:49:54,068 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:49:57,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:02,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:07,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:08,005 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:50:09,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:50:09,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:50:09,068 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:50:13,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:18,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:23,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:24,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:50:24,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:50:24,068 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:50:29,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:34,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:38,007 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:50:39,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:50:39,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:50:39,068 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:50:39,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:44,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:49,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:50:52,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:50:52,414 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:50:52,414 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:50:52,416 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:50:52,758 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:50:53,759 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:50:54,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:50:54,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:50:54,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:50:55,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:00,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:05,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:08,008 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:51:09,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:51:09,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:51:09,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:51:10,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:15,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:20,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:24,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:51:24,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:51:24,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:51:26,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:31,323 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:36,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:38,009 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:51:39,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:51:39,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:51:39,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:51:42,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:47,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:52,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:51:54,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:51:54,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:51:54,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:51:55,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:51:55,137 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:51:55,137 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:51:55,139 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:51:55,802 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:51:55,802 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:51:58,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:03,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:08,010 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:52:09,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:09,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:52:09,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:52:09,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:52:14,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:19,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:24,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:52:24,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:52:24,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:52:24,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:29,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:34,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:38,011 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:52:39,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:52:39,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:52:39,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:52:40,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:45,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:50,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:54,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:52:54,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:52:54,069 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:52:55,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:52:57,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:52:57,926 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:52:57,926 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:52:57,929 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:52:58,845 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:52:59,846 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:53:00,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:05,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:08,014 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:53:09,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:53:09,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:53:09,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:53:11,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:16,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:21,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:24,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:53:24,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:53:24,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:53:26,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:31,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:36,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:38,017 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:53:39,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:53:39,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:53:39,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:53:41,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:46,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:51,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:53:54,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:53:54,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:53:54,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:53:57,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:00,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:54:00,954 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:54:00,954 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:54:00,956 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:54:01,887 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:54:02,888 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:54:02,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:07,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:08,016 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:54:09,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:54:09,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:54:09,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:54:13,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:18,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:23,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:24,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:54:24,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:54:24,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:54:28,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:33,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:38,018 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:54:39,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:39,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:54:39,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:54:39,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:54:44,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:49,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:54,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:54:54,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:54:54,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:54:54,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:54:59,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:03,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:55:03,968 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:55:03,969 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:55:03,971 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:55:04,929 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:55:04,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:06,930 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:55:08,018 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:55:09,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:55:09,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:55:09,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:55:10,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:15,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:20,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:24,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:55:24,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:55:24,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:55:25,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:30,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:35,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:38,019 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:55:39,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:55:39,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:55:39,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:55:41,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:46,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:51,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:55:54,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:55:54,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:55:54,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:55:56,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:01,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:06,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:06,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:56:06,473 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:56:06,473 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:56:06,475 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:56:06,975 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:56:08,020 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:56:08,976 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:56:09,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:56:09,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:56:09,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:56:12,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:17,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:22,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:24,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:56:24,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:56:24,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:56:28,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:33,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:38,021 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:56:39,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:39,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:56:39,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:56:39,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:56:44,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:49,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:54,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:56:54,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:56:54,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:56:54,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:56:59,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:04,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:08,022 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:57:09,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:57:09,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:57:09,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:57:09,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:57:09,790 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:57:09,790 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:57:09,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:09,793 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:57:10,016 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:57:13,018 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:57:14,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:19,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:24,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:57:24,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:57:24,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:57:25,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:30,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:35,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:38,023 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:57:39,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:57:39,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:57:39,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:57:40,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:45,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:50,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:57:54,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:57:54,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:57:54,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:57:55,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:00,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:05,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:08,024 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:58:09,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:58:09,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:58:09,071 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:58:11,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:13,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:58:13,180 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:58:13,180 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:58:13,181 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:58:14,064 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:58:15,065 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:58:16,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:21,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:24,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:58:24,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:58:24,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:58:26,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:31,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:36,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:38,025 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:58:39,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:58:39,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:58:39,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:58:41,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:46,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:51,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:58:54,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:58:54,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:58:54,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:58:56,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:01,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:06,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:08,026 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:59:09,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:59:09,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:59:09,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:59:12,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:16,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 23:59:16,380 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-10-31 23:59:16,380 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-10-31 23:59:16,382 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 23:59:17,110 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-10-31 23:59:17,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:19,112 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-10-31 23:59:22,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:24,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:59:24,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:59:24,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:59:28,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:33,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:38,027 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-10-31 23:59:39,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:39,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:59:39,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:59:39,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:59:44,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:49,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 23:59:54,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 23:59:54,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 23:59:54,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-10-31 23:59:55,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:00,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:05,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:08,029 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:00:09,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:00:09,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:00:09,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:00:11,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:16,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:19,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:00:19,106 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:00:19,106 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:00:19,108 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:00:19,176 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:00:19,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:00:21,177 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:00:21,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:24,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:00:24,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:00:24,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:00:27,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:32,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:37,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:38,030 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:00:39,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:00:39,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:00:39,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:00:43,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:48,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:53,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:00:54,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:00:54,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:00:54,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:00:59,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:04,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:08,031 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:01:09,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:01:09,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:01:09,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:01:09,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:14,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:19,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:22,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:01:22,573 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:01:22,574 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:01:22,575 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:01:23,221 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:01:24,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:01:24,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:01:24,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:01:24,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:25,222 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:01:29,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:34,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:38,032 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:01:39,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:01:39,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:01:39,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:01:39,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:44,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:49,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:01:54,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:01:54,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:01:54,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:01:55,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:00,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:05,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:08,033 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:02:09,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:02:09,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:02:09,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:02:10,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:15,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:20,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:24,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:02:24,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:02:24,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:02:25,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:02:25,653 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:02:25,654 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:02:25,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:25,655 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:02:26,265 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:02:27,266 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:02:30,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:35,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:38,034 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:02:39,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:02:39,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:02:39,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:02:41,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:46,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:51,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:02:54,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:02:54,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:02:54,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:02:57,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:02,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:07,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:08,035 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:03:09,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:03:09,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:03:09,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:03:12,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:17,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:22,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:24,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:03:24,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:03:24,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:03:27,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:28,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:03:28,479 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:03:28,480 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:03:28,482 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:03:29,303 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:03:31,305 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:03:32,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:37,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:38,035 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:03:39,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:03:39,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:03:39,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:03:43,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:48,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:53,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:03:54,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:03:54,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:03:54,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:03:58,323 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:03,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:08,036 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:04:09,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:09,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:04:09,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:04:09,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:04:14,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:19,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:24,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:04:24,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:04:24,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:04:25,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:30,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:31,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:04:31,244 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:04:31,244 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:04:31,246 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:04:31,361 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:04:33,362 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:04:36,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:38,037 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:04:39,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:04:39,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:04:39,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:04:41,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:46,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:51,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:04:54,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:04:54,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:04:54,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:04:56,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:01,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:06,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:08,038 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:05:09,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:05:09,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:05:09,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:05:11,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:16,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:21,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:24,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:05:24,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:05:24,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:05:27,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:32,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:33,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:05:33,556 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:05:33,556 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:05:33,558 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:05:34,438 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:05:35,438 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:05:37,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:38,039 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:05:39,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:05:39,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:05:39,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:05:43,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:48,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:53,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:05:54,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:05:54,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:05:54,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:05:59,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:04,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:08,040 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:06:09,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:06:09,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:06:09,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:06:10,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:15,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:20,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:24,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:06:24,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:06:24,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:06:26,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:31,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:35,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:06:35,898 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:06:35,898 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:06:35,900 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:06:36,482 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:06:36,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:37,483 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:06:38,041 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:06:39,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:06:39,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:06:39,074 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:06:42,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:47,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:52,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:06:54,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:06:54,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:06:54,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:06:58,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:03,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:08,042 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:07:09,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:09,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:07:09,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:07:09,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:07:14,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:19,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:24,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:07:24,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:07:24,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:07:24,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:29,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:34,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:38,042 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:07:38,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:07:38,311 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:07:38,311 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:07:38,312 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:07:38,520 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:07:39,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:07:39,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:07:39,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:07:39,520 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:07:40,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:45,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:50,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:07:54,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:07:54,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:07:54,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:07:55,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:00,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:05,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:08,043 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:08:09,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:08:09,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:08:09,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:08:10,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:15,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:20,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:24,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:08:24,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:08:24,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:08:25,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:30,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:35,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:38,044 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:08:39,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:08:39,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:08:39,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:08:40,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:40,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:08:40,669 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:08:40,669 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:08:40,671 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:08:41,563 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:08:43,564 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:08:45,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:50,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:08:54,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:08:54,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:08:54,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:08:56,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:01,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:06,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:08,045 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:09:09,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:09:09,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:09:09,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:09:12,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:17,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:22,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:24,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:09:24,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:09:24,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:09:27,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:32,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:37,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:38,046 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:09:39,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:09:39,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:09:39,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:09:43,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:43,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:09:43,609 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:09:43,609 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:09:43,610 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:09:44,603 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:09:45,603 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:09:48,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:53,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:09:54,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:09:54,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:09:54,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:09:59,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:04,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:08,047 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:10:09,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:10:09,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:10:09,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:10:10,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:15,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:20,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:24,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:10:24,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:10:24,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:10:25,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:30,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:35,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:38,048 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:10:39,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:10:39,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:10:39,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:10:40,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:45,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:45,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:10:45,904 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:10:45,904 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:10:45,906 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:10:46,664 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:10:47,664 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:10:50,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:10:54,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:10:54,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:10:54,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:10:56,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:01,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:06,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:08,049 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:11:09,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:11:09,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:11:09,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:11:11,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:16,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:21,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:24,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:11:24,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:11:24,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:11:27,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:32,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:37,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:38,050 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:11:39,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:11:39,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:11:39,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:11:42,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:47,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:48,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:11:48,349 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:11:48,349 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:11:48,350 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:11:48,701 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:11:49,701 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:11:52,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:11:54,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:11:54,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:11:54,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:11:58,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:03,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:08,051 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:12:09,051 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:09,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:12:09,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:12:09,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:12:14,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:19,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:24,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:12:24,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:12:24,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:12:25,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:30,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:35,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:38,052 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:12:39,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:12:39,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:12:39,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:12:40,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:45,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:50,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:12:51,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:12:51,088 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:12:51,088 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:12:51,089 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:12:51,742 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:12:53,743 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:12:54,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:12:54,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:12:54,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:12:56,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:01,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:06,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:08,052 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:13:09,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:13:09,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:13:09,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:13:11,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:16,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:21,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:24,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:13:24,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:13:24,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:13:26,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:31,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:36,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:38,053 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:13:39,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:13:39,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:13:39,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:13:42,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:47,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:52,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:13:53,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:13:53,562 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:13:53,562 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:13:53,564 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:13:53,780 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:13:54,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:13:54,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:13:54,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:13:55,781 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:13:57,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:02,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:07,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:08,054 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:14:09,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:14:09,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:14:09,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:14:12,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:17,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:22,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:24,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:14:24,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:14:24,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:14:27,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:32,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:37,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:38,055 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:14:39,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:14:39,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:14:39,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:14:43,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:48,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:53,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:14:54,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:14:54,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:14:54,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:14:56,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:14:56,092 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:14:56,093 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:14:56,094 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:14:56,824 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:14:57,824 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:14:59,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:04,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:08,056 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:15:09,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:15:09,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:15:09,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:15:09,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:14,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:19,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:24,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:15:24,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:15:24,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:15:24,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:29,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:34,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:38,057 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:15:39,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:15:39,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:15:39,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:15:40,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:45,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:50,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:54,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:15:54,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:15:54,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:15:55,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:15:58,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:15:58,784 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:15:58,784 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:15:58,785 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:15:58,864 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:15:59,864 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:16:00,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:05,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:08,058 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:16:09,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:16:09,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:16:09,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:16:11,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:16,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:21,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:24,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:16:24,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:16:24,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:16:26,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:31,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:36,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:38,060 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:16:39,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:16:39,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:16:39,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:16:41,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:46,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:51,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:16:54,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:16:54,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:16:54,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:16:57,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:01,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:17:01,711 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:17:01,712 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:17:01,713 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:17:01,899 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:17:02,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:03,900 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:17:07,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:08,059 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:17:09,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:17:09,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:17:09,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:17:13,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:18,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:23,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:24,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:17:24,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:17:24,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:17:28,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:33,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:38,060 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:17:39,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:39,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:17:39,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:17:39,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:17:44,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:49,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:54,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:17:54,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:17:54,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:17:54,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:17:59,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:03,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:18:03,989 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:18:03,990 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:18:03,991 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:18:04,938 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:18:04,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:05,938 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:18:08,061 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:18:09,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:18:09,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:18:09,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:18:10,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:15,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:20,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:24,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:18:24,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:18:24,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:18:26,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:31,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:36,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:38,062 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:18:39,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:18:39,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:18:39,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:18:41,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:46,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:51,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:18:54,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:18:54,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:18:54,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:18:56,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:01,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:06,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:19:06,248 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:19:06,248 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:19:06,249 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:19:06,979 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:19:07,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:07,980 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:19:08,063 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:19:09,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:19:09,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:19:09,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:19:12,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:17,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:22,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:24,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:19:24,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:19:24,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:19:28,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:33,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:38,064 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:19:39,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:39,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:19:39,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:19:39,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:19:44,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:49,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:54,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:19:54,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:19:54,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:19:54,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:19:59,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:04,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:08,065 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:20:09,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:20:09,061 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:20:09,061 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:20:09,063 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:20:09,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:20:09,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:20:09,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:20:10,016 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:20:10,016 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:20:10,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:15,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:20,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:24,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:20:24,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:20:24,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:20:25,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:30,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:35,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:38,066 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:20:39,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:20:39,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:20:39,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:20:41,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:46,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:51,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:20:54,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:20:54,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:20:54,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:20:56,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:01,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:06,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:08,067 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:21:09,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:21:09,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:21:09,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:21:11,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:21:11,273 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:21:11,274 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:21:11,275 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:21:11,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:11,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:21:12,062 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:21:14,063 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:21:16,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:21,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:24,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:21:24,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:21:24,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:21:27,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:32,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:37,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:38,068 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:21:39,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:21:39,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:21:39,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:21:43,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:48,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:53,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:21:54,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:21:54,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:21:54,153 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:21:58,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:03,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:08,069 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:22:09,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:09,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:22:09,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:22:09,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:22:14,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:14,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:22:14,847 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:22:14,847 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:22:14,849 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:22:15,105 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:22:16,106 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:22:19,850 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:24,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:22:24,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:22:24,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:22:25,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:30,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:35,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:38,069 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:22:39,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:22:39,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:22:39,152 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:22:40,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:45,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:50,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:22:54,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:22:54,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:22:54,153 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:22:55,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:00,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:05,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:08,070 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:23:09,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:23:09,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:23:09,153 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:23:10,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:15,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:17,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:23:17,164 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:23:17,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:23:17,165 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:23:18,142 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:23:18,142 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:23:21,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:24,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:23:24,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:23:24,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:23:26,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:31,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:36,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:38,071 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:23:39,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:23:39,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:23:39,153 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:23:42,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:47,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:52,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:23:54,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:23:54,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:23:54,153 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:23:57,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:02,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:07,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:08,072 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:24:09,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:24:09,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:24:09,153 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:24:12,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:17,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:19,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:24:19,512 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:24:19,512 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:24:19,514 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:24:20,232 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:24:22,234 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:24:22,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:24,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:24:24,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:24:24,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:24:28,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:33,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:38,073 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:24:39,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:39,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:24:39,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:24:39,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:24:44,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:49,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:54,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:24:54,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:24:54,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:24:54,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:24:59,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:04,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:08,074 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:25:09,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:25:09,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:25:09,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:25:10,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:15,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:20,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:21,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:25:21,966 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:25:21,966 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:25:21,968 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:25:22,304 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:25:24,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:25:24,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:25:24,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:25:24,305 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:25:25,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:30,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:35,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:38,075 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:25:39,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:25:39,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:25:39,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:25:41,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:46,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:51,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:25:54,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:25:54,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:25:54,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:25:56,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:01,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:06,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:08,076 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:26:09,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:26:09,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:26:09,154 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:26:12,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:17,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:22,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:24,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:26:24,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:26:24,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:26:24,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:26:24,350 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:26:24,350 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:26:24,352 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:26:25,341 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:26:26,342 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:26:28,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:33,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:38,077 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:26:39,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:39,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:26:39,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:26:39,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:26:44,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:49,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:54,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:26:54,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:26:54,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:26:54,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:26:59,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:04,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:08,078 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:27:09,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:27:09,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:27:09,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:27:09,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:14,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:19,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:24,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:27:24,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:27:24,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:27:25,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:26,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:27:26,871 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:27:26,872 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:27:26,873 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:27:27,399 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:27:28,399 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:27:30,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:35,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:38,078 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:27:39,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:27:39,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:27:39,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:27:41,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:46,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:51,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:27:54,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:27:54,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:27:54,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:27:57,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:02,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:07,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:08,079 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:28:09,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:28:09,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:28:09,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:28:13,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:18,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:23,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:24,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:28:24,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:28:24,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:28:28,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:29,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:28:29,509 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:28:29,509 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:28:29,510 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:28:30,435 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:28:30,435 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:28:33,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:38,080 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:28:39,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:39,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:28:39,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:28:39,155 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:28:44,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:49,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:54,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:28:54,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:28:54,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:28:54,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:28:59,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:04,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:08,081 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:29:09,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:29:09,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:29:09,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:29:10,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:15,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:20,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:24,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:29:24,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:29:24,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:29:26,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:31,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:32,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:29:32,249 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:29:32,249 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:29:32,250 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:29:32,470 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:29:34,471 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:29:37,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:38,082 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:29:39,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:29:39,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:29:39,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:29:42,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:47,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:52,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:29:54,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:29:54,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:29:54,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:29:57,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:02,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:07,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:08,083 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:30:09,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:30:09,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:30:09,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:30:13,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:18,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:23,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:24,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:30:24,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:30:24,156 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:30:29,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:34,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:34,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:30:34,375 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:30:34,375 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:30:34,377 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:30:34,504 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:30:36,505 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:30:38,084 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:30:39,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:30:39,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:30:39,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:30:39,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:44,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:49,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:30:54,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:30:54,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:30:54,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:30:55,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:00,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:05,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:08,085 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:31:09,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:31:09,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:31:09,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:31:11,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:16,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:21,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:24,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:31:24,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:31:24,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:31:26,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:31,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:36,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:36,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:31:36,790 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:31:36,790 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:31:36,792 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:31:37,539 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:31:38,088 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:31:38,539 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:31:39,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:31:39,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:31:39,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:31:42,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:47,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:52,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:31:54,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:31:54,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:31:54,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:31:58,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:03,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:08,086 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:32:09,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:09,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:32:09,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:32:09,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:32:14,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:19,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:24,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:32:24,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:32:24,157 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:32:25,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:30,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:35,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:38,088 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:32:39,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:32:39,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:32:39,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:32:39,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:32:39,671 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:32:39,672 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:32:39,673 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:32:40,572 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:32:40,572 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:32:40,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:45,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:50,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:32:54,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:32:54,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:32:54,160 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:32:56,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:01,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:06,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:08,088 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:33:09,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:33:09,160 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:33:09,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:33:11,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:16,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:21,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:24,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:33:24,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:33:24,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:33:27,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:32,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:37,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:38,089 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:33:39,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:33:39,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:33:39,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:33:42,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:33:42,108 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:33:42,109 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:33:42,110 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:33:42,606 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:33:43,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:44,607 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:33:48,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:53,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:33:54,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:33:54,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:33:54,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:33:58,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:03,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:08,090 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:34:09,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:09,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:34:09,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:34:09,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:34:14,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:19,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:24,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:34:24,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:34:24,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:34:24,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:29,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:34,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:38,091 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:34:39,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:34:39,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:34:39,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:34:40,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:44,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:34:44,353 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:34:44,353 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:34:44,355 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:34:44,638 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:34:45,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:46,639 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:34:50,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:34:54,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:34:54,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:34:54,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:34:56,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:01,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:06,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:08,092 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:35:09,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:35:09,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:35:09,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:35:11,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:16,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:21,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:24,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:35:24,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:35:24,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:35:27,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:32,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:37,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:38,093 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:35:39,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:35:39,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:35:39,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:35:43,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:46,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:35:46,795 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:35:46,795 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:35:46,797 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:35:47,671 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:35:48,672 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:35:48,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:53,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:35:54,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:35:54,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:35:54,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:35:59,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:04,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:08,093 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:36:09,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:36:09,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:36:09,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:36:10,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:15,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:20,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:24,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:36:24,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:36:24,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:36:25,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:30,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:35,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:38,094 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:36:39,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:36:39,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:36:39,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:36:41,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:46,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:49,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:36:49,462 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:36:49,462 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:36:49,463 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:36:49,704 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:36:50,705 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:36:51,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:36:54,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:36:54,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:36:54,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:36:57,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:02,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:07,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:08,095 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:37:09,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:37:09,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:37:09,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:37:13,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:18,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:23,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:24,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:37:24,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:37:24,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:37:28,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:33,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:38,096 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:37:39,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:39,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:37:39,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:37:39,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:37:44,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:49,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:51,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:37:51,684 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:37:51,684 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:37:51,685 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:37:51,750 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:37:52,751 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:37:54,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:37:54,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:37:54,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:37:54,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:37:59,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:04,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:08,098 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:38:09,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:38:09,162 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:38:09,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:38:10,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:15,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:20,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:24,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:38:24,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:38:24,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:38:25,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:30,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:35,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:38,099 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:38:39,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:38:39,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:38:39,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:38:41,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:46,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:51,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:54,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:38:54,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:38:54,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:38:54,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:38:54,406 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:38:54,406 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:38:54,408 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:38:54,787 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:38:56,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:38:56,788 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:39:01,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:06,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:08,100 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:39:09,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:39:09,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:39:09,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:39:11,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:16,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:21,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:24,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:39:24,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:39:24,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:39:27,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:32,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:37,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:38,103 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:39:39,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:39:39,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:39:39,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:39:43,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:48,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:53,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:39:54,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:39:54,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:39:54,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:39:57,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:39:57,035 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:39:57,035 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:39:57,036 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:39:57,823 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:39:58,823 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:39:59,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:04,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:08,102 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:40:09,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:09,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:40:09,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:40:09,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:40:14,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:19,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:24,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:40:24,163 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:40:24,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:40:25,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:30,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:35,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:38,103 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:40:39,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:40:39,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:40:39,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:40:40,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:45,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:50,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:54,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:40:54,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:40:54,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:40:55,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:40:59,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:40:59,389 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:40:59,389 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:40:59,390 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:40:59,856 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:41:00,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:00,857 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:41:05,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:08,103 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:41:09,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:41:09,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:41:09,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:41:11,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:16,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:21,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:24,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:41:24,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:41:24,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:41:26,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:31,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:36,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:38,105 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:41:39,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:41:39,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:41:39,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:41:42,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:47,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:52,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:41:54,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:41:54,164 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:41:54,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:41:57,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:01,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:42:01,712 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:42:01,712 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:42:01,714 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:42:01,891 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:42:02,891 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:42:03,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:08,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:08,105 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:42:09,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:42:09,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:42:09,686 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:42:12,897 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:42:13,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:18,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:20,902 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:42:21,903 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:42:23,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:24,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:42:24,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:42:24,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:42:29,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:34,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:34,911 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:42:35,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:42:36,912 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:42:38,106 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:42:39,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:39,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:42:39,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:42:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:42:44,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:49,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:42:54,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:42:54,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:42:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:42:55,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:00,859 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:05,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:08,107 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:43:09,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:43:09,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:43:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:43:11,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:16,859 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:21,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:24,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:43:24,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:43:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:43:27,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:32,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:37,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:38,108 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:43:38,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:43:38,674 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:43:38,674 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:43:38,675 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:43:38,952 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:43:39,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:43:39,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:43:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:43:40,954 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:43:42,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:47,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:52,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:43:54,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:43:54,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:43:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:43:57,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:02,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:07,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:08,109 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:44:09,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:44:09,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:44:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:44:13,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:18,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:23,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:24,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:44:24,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:44:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:44:28,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:33,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:38,110 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:44:39,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:39,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:44:39,619 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:44:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:44:41,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:44:41,344 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:44:41,344 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:44:41,345 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:44:41,989 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:44:43,990 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:44:44,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:49,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:54,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:44:54,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:44:54,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:44:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:44:59,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:04,839 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:08,111 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:45:09,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:45:09,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:45:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:45:10,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:15,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:20,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:24,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:45:24,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:45:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:45:25,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:30,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:35,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:38,112 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:45:39,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:45:39,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:45:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:45:40,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:43,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:45:43,839 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:45:43,839 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:45:43,840 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:45:44,024 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:45:46,025 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:45:46,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:51,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:45:54,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:45:54,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:45:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:45:56,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:01,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:06,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:08,113 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:46:09,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:46:09,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:46:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:46:12,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:17,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:22,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:24,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:46:24,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:46:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:46:27,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:32,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:37,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:38,142 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:46:39,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:46:39,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:46:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:46:43,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:45,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:46:45,987 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:46:45,987 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:46:45,988 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:46:46,060 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:46:48,062 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:46:49,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:54,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:46:54,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:46:54,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:46:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:46:59,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:04,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:08,140 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:47:09,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:47:09,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:47:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:47:09,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:14,839 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:19,840 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:24,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:47:24,620 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:47:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:47:25,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:30,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:35,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:38,142 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:47:39,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:47:39,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:47:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:47:40,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:45,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:48,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:47:48,583 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:47:48,583 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:47:48,585 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:47:49,097 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:47:50,098 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:47:51,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:47:54,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:47:54,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:47:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:47:56,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:01,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:06,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:08,142 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:48:09,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:48:09,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:48:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:48:12,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:17,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:22,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:24,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:48:24,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:48:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:48:27,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:32,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:37,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:38,143 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:48:39,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:48:39,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:48:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:48:43,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:48,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:51,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:48:51,100 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:48:51,101 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:48:51,102 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:48:51,133 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:48:54,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:48:54,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:48:54,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:48:54,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:48:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:48:59,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:04,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:08,144 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:49:09,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:49:09,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:49:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:49:10,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:15,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:20,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:24,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:49:24,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:49:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:49:25,848 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:30,848 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:35,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:38,144 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:49:39,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:49:39,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:49:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:49:41,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:46,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:51,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:49:53,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:49:53,832 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:49:53,833 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:49:53,834 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:49:54,172 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:49:54,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:49:54,621 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:49:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:49:56,174 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:49:56,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:01,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:06,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:08,145 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:50:09,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:50:09,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:50:09,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:50:12,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:17,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:22,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:24,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:50:24,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:50:24,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:50:28,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:33,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:38,146 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:50:39,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:39,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:50:39,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:50:39,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:50:45,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:50,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:54,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:50:54,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:50:54,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:50:55,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:50:56,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:50:56,217 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:50:56,217 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:50:56,222 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:50:57,212 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:50:58,213 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:51:01,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:06,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:08,147 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:51:09,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:51:09,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:51:09,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:51:11,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:16,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:21,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:24,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:51:24,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:51:24,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:51:26,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:31,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:36,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:38,148 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:51:39,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:51:39,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:51:39,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:51:41,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:46,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:51,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:54,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:51:54,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:51:54,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:51:57,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:51:58,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:51:58,782 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:51:58,782 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:51:58,784 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:51:59,252 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:52:00,252 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:52:03,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:08,149 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:52:09,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:09,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:52:09,622 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:52:09,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:52:14,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:19,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:24,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 00:52:24,623 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: stop_status +2024-11-01 00:52:24,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:52:30,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:52:35,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:52:39,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:52:44,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:52:45,981 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:52:45,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:49,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:52:50,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:54,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:52:55,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:52:59,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:00,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:01,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:53:01,545 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:53:01,545 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:53:01,546 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:53:02,288 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:53:04,289 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:53:04,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:06,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:08,151 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:53:09,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:53:12,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:14,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:17,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:19,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:22,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:24,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:53:27,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:29,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:32,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:34,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:37,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:38,153 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:53:39,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:53:43,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:44,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:48,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:49,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:53:53,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:54,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:53:58,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:53:59,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:03,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:03,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:54:03,717 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:54:03,718 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:54:03,719 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:54:04,324 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:54:04,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:06,325 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:54:08,155 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:54:09,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:09,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:54:14,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:14,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:19,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:19,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:24,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:24,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:54:29,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:29,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:34,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:34,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:38,154 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:54:39,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:54:40,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:44,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:45,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:49,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:54:50,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:54,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:54:55,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:54:59,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:00,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:04,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:05,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:06,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:55:06,269 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:55:06,269 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:55:06,272 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:55:06,401 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:55:08,155 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:55:08,402 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:55:09,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:55:11,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:14,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:16,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:19,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:21,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:24,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:55:26,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:29,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:31,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:34,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:36,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:38,155 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:55:39,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:55:42,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:44,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:47,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:49,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:55:52,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:54,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:55:57,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:55:59,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:02,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:04,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:07,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:08,156 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:56:08,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:56:08,536 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:56:08,537 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:56:08,546 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:56:09,438 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:56:09,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:56:10,439 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:56:12,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:14,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:17,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:19,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:22,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:24,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:24,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:56:27,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:29,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:32,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:34,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:37,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:38,157 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:56:39,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:56:43,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:44,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:48,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:49,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:56:53,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:54,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:56:58,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:56:59,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:03,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:04,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:08,158 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:57:09,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:09,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:57:11,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:57:11,051 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:57:11,051 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:57:11,052 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:57:11,476 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:57:12,476 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:57:14,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:15,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:19,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:20,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:24,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:57:25,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:29,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:30,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:34,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:35,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:38,159 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:57:39,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:57:40,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:44,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:45,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:49,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:50,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:54,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:57:54,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:57:55,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:57:59,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:00,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:04,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:05,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:08,160 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:58:09,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:58:11,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:13,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:58:13,567 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:58:13,567 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:58:13,568 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:58:14,515 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:58:14,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:16,516 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:58:16,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:19,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:21,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:24,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:58:26,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:29,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:31,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:34,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:36,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:38,161 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:58:39,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:58:42,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:44,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:47,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:49,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:58:52,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:54,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:58:57,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:58:59,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:02,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:04,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:07,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:08,161 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:59:09,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:59:13,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:14,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:16,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 00:59:16,440 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 00:59:16,440 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 00:59:16,442 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 00:59:16,551 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 00:59:18,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:18,552 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 00:59:19,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:23,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:24,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:59:28,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:29,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:33,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:34,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:38,162 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 00:59:39,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:39,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:59:44,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:44,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:49,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:49,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 00:59:54,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:54,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 00:59:59,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 00:59:59,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:04,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:04,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:08,163 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:00:09,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:00:10,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:14,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:15,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:18,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:00:18,559 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:00:18,559 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:00:18,561 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:00:18,586 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:00:19,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:20,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:20,587 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:00:24,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:00:25,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:29,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:30,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:34,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:35,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:38,164 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:00:39,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:00:41,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:44,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:46,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:49,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:00:51,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:54,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:00:56,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:00:59,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:01,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:04,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:06,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:08,165 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:01:09,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:01:12,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:14,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:17,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:19,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:21,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:01:21,562 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:01:21,563 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:01:21,564 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:01:21,667 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:01:22,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:22,667 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:01:24,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:01:27,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:29,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:32,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:34,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:37,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:38,166 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:01:39,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:01:43,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:44,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:48,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:49,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:01:53,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:54,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:01:58,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:01:59,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:03,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:04,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:08,170 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:02:09,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:09,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:02:14,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:14,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:19,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:19,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:23,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:02:23,670 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:02:23,671 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:02:23,672 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:02:23,703 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:02:24,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:24,703 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:02:24,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:02:29,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:30,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:34,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:35,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:38,168 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:02:39,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:02:40,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:45,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:45,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:50,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:02:50,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:02:54,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:02:55,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:00,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:00,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:05,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:05,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:08,169 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:03:09,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:03:11,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:15,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:16,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:20,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:21,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:24,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:03:26,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:26,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:03:26,234 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:03:26,234 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:03:26,235 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:03:26,775 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:03:27,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:03:28,776 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:03:30,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:32,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:35,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:37,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:38,170 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:03:39,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:03:42,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:45,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:47,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:50,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:03:52,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:03:54,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:03:57,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:00,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:02,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:05,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:07,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:08,171 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:04:09,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:04:13,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:15,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:18,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:20,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:23,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:24,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:04:28,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:29,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:04:29,311 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:04:29,312 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:04:29,313 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:04:29,811 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:04:30,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:30,812 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:04:33,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:35,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:38,172 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:04:39,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:39,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:04:44,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:45,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:49,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:50,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:04:54,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:04:54,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:04:59,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:00,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:04,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:05,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:08,173 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:05:09,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:05:10,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:15,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:15,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:20,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:20,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:24,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:05:25,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:30,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:30,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:32,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:05:32,078 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:05:32,078 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:05:32,080 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:05:32,853 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:05:34,854 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:05:35,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:36,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:38,175 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:05:39,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:05:41,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:45,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:46,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:50,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:05:51,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:05:54,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:05:56,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:00,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:01,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:05,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:06,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:08,176 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:06:09,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:06:12,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:15,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:17,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:20,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:22,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:24,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:06:27,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:30,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:32,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:34,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:06:34,458 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:06:34,459 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:06:34,460 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:06:34,935 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:06:35,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:36,936 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:06:37,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:38,176 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:06:39,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:06:43,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:45,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:48,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:50,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:06:53,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:06:54,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:06:58,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:00,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:03,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:05,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:08,177 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:07:09,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:09,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:07:14,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:15,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:19,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:20,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:24,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:24,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:07:29,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:30,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:34,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:35,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:37,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:07:37,140 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:07:37,141 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:07:37,142 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:07:38,026 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:07:38,178 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:07:39,026 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:07:39,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:07:40,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:45,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:45,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:50,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:07:50,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:07:54,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:07:55,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:00,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:00,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:05,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:05,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:08,179 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:08:09,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:08:11,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:15,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:16,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:20,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:21,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:24,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:08:26,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:30,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:31,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:35,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:36,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:38,180 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:08:39,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:08:40,047 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:08:40,048 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:08:40,049 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:08:40,091 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:08:41,063 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:08:41,063 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:08:42,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:45,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:47,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:50,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:08:52,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:08:54,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:08:57,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:00,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:02,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:05,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:07,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:08,181 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:09:09,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:09:12,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:15,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:17,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:20,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:22,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:24,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:09:27,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:30,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:32,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:35,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:37,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:38,184 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:09:39,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:09:42,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:42,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:09:42,573 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:09:42,573 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:09:42,575 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:09:43,099 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:09:45,100 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:09:45,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:47,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:50,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:09:52,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:09:54,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:09:57,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:00,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:02,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:05,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:07,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:08,183 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:10:09,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:10:13,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:15,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:18,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:20,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:23,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:24,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:10:28,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:30,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:33,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:35,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:38,185 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:10:39,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:39,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:10:44,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:45,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:10:45,139 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:10:45,139 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:10:45,141 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:10:45,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:46,134 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:10:47,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:10:50,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:10:50,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:10:54,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:10:55,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:00,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:00,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:05,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:05,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:08,185 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:11:09,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:11:11,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:15,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:16,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:20,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:21,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:24,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:11:26,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:30,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:31,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:35,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:36,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:38,186 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:11:39,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:11:42,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:45,515 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:47,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:47,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:11:47,499 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:11:47,499 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:11:47,501 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:11:48,169 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:11:49,169 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:11:50,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:11:52,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:11:54,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:11:57,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:00,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:02,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:05,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:07,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:08,186 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:12:09,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:12:13,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:15,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:18,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:20,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:23,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:24,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:12:28,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:30,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:33,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:35,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:38,189 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:12:39,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:39,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:12:44,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:45,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:49,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:49,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:12:49,675 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:12:49,676 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:12:49,677 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:12:50,217 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:12:50,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:12:51,217 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:12:54,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:12:54,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:12:59,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:00,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:04,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:05,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:08,189 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:13:09,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:13:10,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:15,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:15,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:20,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:20,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:24,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:13:25,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:30,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:30,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:35,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:35,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:38,190 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:13:39,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:13:41,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:45,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:46,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:50,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:13:51,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:13:52,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:13:52,253 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:13:52,254 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:13:52,256 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:13:52,291 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:13:53,291 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:13:54,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:13:56,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:00,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:01,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:05,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:06,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:08,191 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:14:09,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:14:12,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:15,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:17,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:20,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:22,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:24,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:14:27,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:30,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:32,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:35,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:37,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:38,192 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:14:39,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:14:43,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:45,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:48,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:50,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:14:53,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:14:54,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:14:54,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:14:54,866 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:14:54,866 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:14:54,868 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:14:55,326 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:14:57,327 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:14:58,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:00,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:03,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:05,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:08,193 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:15:09,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:09,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:15:14,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:15,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:19,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:20,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:24,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:24,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:15:29,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:30,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:34,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:35,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:38,193 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:15:39,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:15:40,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:45,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:45,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:50,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:50,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:15:54,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:15:55,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:15:57,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:15:57,807 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:15:57,808 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:15:57,809 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:15:58,376 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:15:59,377 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:16:00,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:00,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:05,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:05,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:08,194 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:16:09,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:16:11,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:15,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:16,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:20,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:21,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:24,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:16:26,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:30,588 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:31,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:35,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:36,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:38,195 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:16:39,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:16:42,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:45,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:47,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:50,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:16:52,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:16:54,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:16:57,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:00,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:17:00,180 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:17:00,180 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:17:00,181 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:17:00,413 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:17:00,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:01,413 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:17:02,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:05,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:07,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:08,198 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:17:09,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:17:13,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:15,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:18,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:20,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:23,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:24,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:17:28,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:30,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:33,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:35,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:38,197 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:17:39,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:39,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:17:44,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:45,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:49,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:50,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:17:54,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:17:54,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:17:59,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:00,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:02,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:18:02,813 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:18:02,813 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:18:02,816 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:18:03,449 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:18:04,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:05,450 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:18:05,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:08,198 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:18:09,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:18:10,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:15,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:15,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:20,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:20,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:24,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:18:25,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:30,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:30,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:35,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:35,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:38,200 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:18:39,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:18:41,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:45,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:46,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:50,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:18:51,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:18:54,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:18:56,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:00,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:01,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:04,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:19:04,989 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:19:04,989 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:19:04,990 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:19:05,483 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:19:05,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:06,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:07,485 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:19:08,199 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:19:09,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:19:12,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:15,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:17,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:20,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:22,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:24,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:19:27,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:30,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:32,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:35,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:37,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:38,200 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:19:39,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:19:43,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:45,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:48,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:50,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:19:53,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:19:54,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:19:58,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:00,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:03,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:05,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:07,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:20:07,330 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:20:07,330 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:20:07,332 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:20:07,519 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:20:08,201 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:20:09,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:09,520 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:20:09,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:20:14,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:15,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:19,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:20,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:24,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:24,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:20:29,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:30,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:34,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:35,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:38,202 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:20:39,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:20:40,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:45,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:45,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:50,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:20:50,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:20:54,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:20:55,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:00,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:00,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:05,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:05,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:08,203 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:21:09,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:21:09,740 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:21:09,741 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:21:09,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:21:09,743 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:21:10,556 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:21:10,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:11,556 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:21:15,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:15,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:20,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:20,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:24,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:21:25,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:30,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:30,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:35,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:35,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:38,204 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:21:39,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:21:41,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:44,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:46,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:49,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:21:51,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:54,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:21:56,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:21:59,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:01,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:04,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:06,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:08,205 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:22:09,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:22:12,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:22:12,200 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:22:12,200 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:22:12,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:12,201 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:22:12,592 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:22:13,592 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:22:14,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:17,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:19,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:22,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:24,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:22:27,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:29,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:32,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:34,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:37,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:38,206 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:22:39,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:22:43,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:44,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:48,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:49,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:22:53,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:54,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:22:58,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:22:59,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:03,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:04,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:08,207 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:23:09,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:09,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:23:14,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:14,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:23:14,755 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:23:14,755 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:23:14,756 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:23:14,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:15,627 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:23:15,627 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:23:19,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:19,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:24,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:23:24,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:29,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:29,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:34,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:34,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:38,207 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:23:39,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:23:40,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:44,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:45,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:49,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:23:50,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:54,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:23:55,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:23:59,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:00,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:04,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:05,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:08,208 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:24:09,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:24:11,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:14,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:16,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:16,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:24:16,958 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:24:16,958 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:24:16,960 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:24:17,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:24:17,662 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:24:17,662 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:24:19,663 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:24:19,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:21,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:24,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:24:26,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:29,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:31,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:34,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:36,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:38,211 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:24:39,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:24:42,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:44,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:47,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:49,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:24:52,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:24:54,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:24:57,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:00,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:02,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:05,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:07,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:08,210 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:25:09,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:25:13,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:15,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:18,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:20,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:20,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:25:20,210 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:25:20,210 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:25:20,212 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:25:20,697 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:25:21,698 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:25:23,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:24,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:25:28,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:30,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:33,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:35,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:38,211 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:25:39,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:39,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:25:44,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:45,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:49,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:50,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:25:54,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:25:54,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:25:59,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:00,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:04,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:05,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:08,213 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:26:09,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:26:10,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:15,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:15,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:20,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:20,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:22,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:26:22,384 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:26:22,384 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:26:22,386 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:26:22,732 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:26:23,733 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:26:24,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:26:25,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:30,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:30,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:35,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:35,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:38,213 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:26:39,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:26:41,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:45,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:46,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:50,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:26:51,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:26:54,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:26:56,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:00,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:01,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:05,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:06,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:08,214 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:27:09,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:27:12,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:15,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:17,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:20,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:22,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:24,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:27:24,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:27:24,903 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:27:24,903 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:27:24,904 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:27:25,768 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:27:25,768 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:27:27,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:30,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:32,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:35,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:37,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:38,214 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:27:39,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:27:43,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:45,047 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:48,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:50,048 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:27:53,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:27:54,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:27:58,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:00,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:03,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:05,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:08,215 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:28:09,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:09,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:28:14,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:15,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:19,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:20,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:24,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:24,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:28:27,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:28:27,582 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:28:27,582 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:28:27,583 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:28:27,803 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:28:29,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:29,804 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:28:30,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:34,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:35,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:38,216 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:28:39,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:28:40,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:45,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:45,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:50,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:28:50,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:28:54,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:28:55,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:00,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:00,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:05,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:05,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:08,217 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:29:09,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:29:11,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:15,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:16,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:20,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:21,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:24,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:29:26,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:29,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:29:29,689 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:29:29,689 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:29:29,690 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:29:29,839 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:29:30,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:31,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:31,840 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:29:35,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:36,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:38,218 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:29:39,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:29:42,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:45,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:47,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:50,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:29:52,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:29:54,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:29:57,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:00,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:02,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:05,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:07,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:08,219 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:30:09,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:30:13,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:15,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:18,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:20,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:23,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:24,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:30:28,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:30,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:32,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:30:32,195 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:30:32,195 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:30:32,197 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:30:32,876 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:30:33,877 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:30:34,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:35,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:38,220 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:30:39,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:39,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:30:44,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:45,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:49,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:50,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:30:54,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:30:54,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:30:59,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:00,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:04,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:05,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:08,221 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:31:09,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:31:10,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:15,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:15,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:20,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:20,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:24,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:31:25,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:30,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:30,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:34,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:31:34,931 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:31:34,931 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:31:34,932 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:31:35,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:35,912 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:31:35,912 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:31:35,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:38,222 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:31:39,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:31:41,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:45,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:46,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:50,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:31:51,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:31:54,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:31:56,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:00,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:01,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:05,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:06,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:08,223 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:32:09,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:32:12,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:15,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:17,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:20,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:22,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:24,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:32:27,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:30,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:32,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:35,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:37,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:37,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:32:37,367 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:32:37,367 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:32:37,368 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:32:37,948 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:32:38,223 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:32:38,948 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:32:39,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:32:43,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:45,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:48,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:50,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:32:53,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:32:54,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:32:58,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:00,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:03,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:05,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:08,224 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:33:09,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:09,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:33:14,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:15,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:19,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:20,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:24,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:24,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:33:29,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:30,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:34,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:35,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:38,225 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:33:39,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:33:39,596 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:33:39,597 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:33:39,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:39,598 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:33:39,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:33:39,984 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:33:40,984 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:33:44,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:45,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:49,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:50,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:33:54,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:33:54,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:33:59,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:00,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:04,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:05,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:08,226 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:34:09,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:34:10,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:15,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:15,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:20,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:20,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:24,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:34:25,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:30,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:30,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:35,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:35,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:38,227 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:34:39,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:34:41,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:42,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:34:42,447 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:34:42,447 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:34:42,448 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:34:43,021 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:34:45,023 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:34:45,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:46,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:50,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:34:51,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:34:54,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:34:55,029 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:34:57,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:00,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:01,033 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:35:02,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:03,034 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:35:05,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:07,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:08,228 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:35:10,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:35:13,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:15,046 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:35:16,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:17,047 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:35:19,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:21,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:24,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:24,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:35:29,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:30,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:34,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:35,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:38,229 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:35:39,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:39,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:35:44,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:45,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:49,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:50,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:35:54,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:35:54,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:35:59,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:00,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:04,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:05,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:08,230 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:36:09,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:36:10,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:15,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:15,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:18,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:36:18,770 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:36:18,770 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:36:18,772 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:36:19,087 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:36:20,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:20,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:21,088 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:36:24,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:36:25,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:30,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:30,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:35,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:35,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:38,231 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:36:39,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:36:41,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:45,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:46,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:50,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:36:51,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:36:54,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:36:56,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:00,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:01,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:05,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:06,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:08,232 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:37:09,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:37:12,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:15,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:17,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:20,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:21,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:37:21,507 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:37:21,507 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:37:21,509 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:37:22,122 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:37:22,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:23,123 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:37:24,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:37:27,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:30,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:32,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:35,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:37,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:38,233 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:37:39,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:37:43,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:45,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:48,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:50,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:37:53,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:37:54,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:37:58,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:00,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:03,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:05,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:08,233 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:38:09,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:09,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:38:14,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:15,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:19,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:20,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:23,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:38:23,733 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:38:23,733 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:38:23,734 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:38:24,157 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:38:24,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:24,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:38:25,158 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:38:29,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:30,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:34,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:35,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:38,234 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:38:39,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:38:40,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:45,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:45,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:50,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:38:50,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:38:54,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:38:55,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:00,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:00,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:05,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:05,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:08,235 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:39:09,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:39:11,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:15,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:16,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:20,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:21,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:24,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:39:26,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:26,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:39:26,639 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:39:26,639 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:39:26,640 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:39:27,194 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:39:29,195 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:39:30,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:31,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:35,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:36,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:38,238 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:39:39,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:39:42,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:45,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:47,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:50,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:39:52,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:39:54,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:39:57,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:00,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:02,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:05,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:07,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:08,237 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:40:09,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:40:13,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:15,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:18,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:20,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:23,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:24,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:40:28,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:28,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:40:28,738 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:40:28,738 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:40:28,740 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:40:29,235 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:40:30,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:31,236 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:40:33,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:35,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:38,238 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:40:39,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:39,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:40:44,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:45,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:49,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:50,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:40:54,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:40:54,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:40:59,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:00,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:04,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:05,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:08,240 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:41:09,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:41:10,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:15,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:15,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:20,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:20,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:24,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:41:25,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:30,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:30,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:31,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:41:31,299 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:41:31,300 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:41:31,301 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:41:32,276 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:41:33,276 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:41:35,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:35,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:38,240 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:41:39,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:41:41,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:45,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:46,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:50,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:41:51,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:41:54,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:41:56,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:00,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:01,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:05,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:06,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:08,240 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:42:09,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:42:12,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:15,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:17,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:20,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:22,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:24,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:42:27,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:30,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:32,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:33,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:42:33,779 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:42:33,779 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:42:33,780 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:42:34,350 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:42:35,351 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:42:35,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:37,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:38,241 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:42:39,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:42:43,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:45,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:48,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:50,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:42:53,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:42:54,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:42:58,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:00,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:03,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:05,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:08,242 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:43:09,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:09,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:43:14,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:15,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:19,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:20,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:24,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:24,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:43:29,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:30,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:34,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:35,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:36,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:43:36,513 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:43:36,513 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:43:36,514 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:43:37,386 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:43:38,243 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:43:39,388 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:43:39,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:43:40,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:45,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:45,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:50,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:43:50,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:43:54,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:43:55,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:00,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:00,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:05,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:05,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:08,244 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:44:09,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:44:11,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:15,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:16,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:20,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:21,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:24,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:44:26,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:30,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:31,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:35,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:36,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:38,245 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:44:39,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:44:39,012 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:44:39,012 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:44:39,014 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:44:39,424 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:44:39,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:44:41,425 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:44:42,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:45,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:47,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:50,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:44:52,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:44:54,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:44:57,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:00,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:02,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:05,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:07,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:08,246 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:45:09,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:45:12,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:15,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:17,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:20,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:22,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:24,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:45:27,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:30,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:32,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:35,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:37,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:38,247 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:45:39,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:45:41,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:45:41,791 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:45:41,791 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:45:41,792 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:45:42,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:42,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:45:42,465 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:45:43,465 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:45:45,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:47,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:50,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:45:52,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:45:54,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:45:57,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:00,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:02,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:05,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:07,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:08,248 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:46:09,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:46:13,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:15,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:18,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:20,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:23,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:24,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:46:28,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:30,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:33,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:35,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:38,249 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:46:39,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:39,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:46:44,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:44,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:46:44,603 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:46:44,603 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:46:44,604 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:46:45,481 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:45,539 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:46:47,540 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:46:49,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:50,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:46:54,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:46:54,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:46:59,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:00,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:04,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:05,485 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:08,252 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:47:09,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:47:10,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:15,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:15,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:20,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:20,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:24,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:47:25,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:30,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:30,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:35,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:35,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:38,250 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:47:39,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:47:41,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:45,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:46,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:47,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:47:47,431 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:47:47,431 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:47:47,433 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:47:47,639 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:47:49,641 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:47:50,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:47:51,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:47:54,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:47:56,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:00,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:01,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:05,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:06,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:08,251 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:48:09,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:48:12,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:15,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:17,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:20,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:22,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:24,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:48:27,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:30,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:32,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:35,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:37,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:38,252 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:48:39,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:48:43,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:45,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:48,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:49,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:48:49,819 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:48:49,820 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:48:49,822 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:48:50,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:48:50,714 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:48:51,714 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:48:53,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:48:54,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:48:58,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:00,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:03,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:05,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:08,253 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:49:09,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:09,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:49:14,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:15,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:19,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:20,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:24,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:24,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:49:29,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:30,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:34,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:35,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:38,254 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:49:39,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:49:40,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:45,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:45,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:50,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:49:50,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:49:52,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:49:52,226 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:49:52,226 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:49:52,227 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:49:52,783 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:49:53,784 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:49:54,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:49:56,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:00,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:01,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:05,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:06,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:08,255 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:50:09,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:50:11,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:15,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:16,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:20,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:21,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:24,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:50:26,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:30,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:31,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:35,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:36,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:38,256 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:50:39,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:50:42,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:45,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:47,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:50,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:50:52,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:50:54,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:50:55,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:50:55,032 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:50:55,032 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:50:55,033 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:50:55,817 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:50:57,818 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:50:58,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:00,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:03,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:05,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:08,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:08,256 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:51:09,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:51:13,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:15,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:18,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:20,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:23,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:24,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:51:28,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:30,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:33,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:35,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:38,257 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:51:39,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:39,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:51:44,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:45,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:49,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:50,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:51:54,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:54,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:51:57,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:51:57,707 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:51:57,708 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:51:57,709 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:51:57,851 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:51:59,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:51:59,853 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:52:00,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:04,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:05,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:08,258 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:52:09,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:52:10,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:15,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:15,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:20,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:20,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:24,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:52:25,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:30,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:30,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:35,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:35,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:38,259 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:52:39,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:52:41,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:45,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:46,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:50,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:52:51,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:52:54,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:52:56,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:00,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:53:00,070 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:53:00,070 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:53:00,071 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:53:00,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:00,887 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:53:01,887 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:53:02,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:05,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:07,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:08,260 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:53:09,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:53:12,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:15,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:17,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:20,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:22,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:24,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:53:27,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:30,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:32,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:35,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:37,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:38,261 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:53:39,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:53:43,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:45,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:48,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:50,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:53:53,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:53:54,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:53:58,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:00,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:02,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:54:02,769 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:54:02,769 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:54:02,771 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:54:02,922 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:54:03,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:03,922 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:54:05,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:08,261 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:54:09,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:09,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:54:14,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:15,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:19,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:20,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:24,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:24,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:54:29,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:30,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:34,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:35,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:38,262 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:54:39,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:54:40,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:45,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:45,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:50,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:54:50,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:54:54,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:54:55,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:00,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:00,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:05,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:55:05,146 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:55:05,146 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:55:05,147 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:55:05,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:05,957 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:55:06,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:07,958 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:55:08,263 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:55:09,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:55:11,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:15,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:16,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:20,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:21,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:24,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:55:26,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:30,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:31,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:35,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:36,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:38,264 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:55:39,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:55:42,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:45,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:47,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:50,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:55:52,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:55:54,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:55:57,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:00,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:02,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:05,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:07,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:07,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:56:07,631 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:56:07,631 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:56:07,632 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:56:07,994 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:56:08,265 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:56:09,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:56:09,995 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:56:13,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:15,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:18,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:20,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:23,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:24,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:56:28,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:30,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:33,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:35,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:38,266 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:56:39,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:39,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:56:44,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:45,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:49,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:50,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:56:54,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:56:54,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:56:59,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:00,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:04,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:05,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:08,267 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:57:09,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:57:10,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:10,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:57:10,289 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:57:10,289 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:57:10,291 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:57:11,031 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:57:12,031 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:57:15,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:15,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:20,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:20,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:24,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:57:25,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:30,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:30,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:35,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:35,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:38,268 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:57:39,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:57:41,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:45,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:46,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:50,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:57:51,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:57:54,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:57:56,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:00,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:01,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:05,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:06,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:08,269 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:58:09,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:58:12,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:12,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:58:12,484 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:58:12,484 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:58:12,485 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:58:13,067 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:58:14,068 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:58:15,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:17,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:20,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:22,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:24,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:58:27,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:30,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:32,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:35,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:37,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:38,270 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:58:39,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:58:43,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:45,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:48,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:50,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:58:53,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:58:54,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:58:58,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:00,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:03,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:05,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:08,270 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:59:09,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:09,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:59:14,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:15,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 01:59:15,178 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 01:59:15,178 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 01:59:15,180 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 01:59:15,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:16,103 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 01:59:16,103 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 01:59:20,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:20,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:24,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:59:25,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:30,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:30,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:35,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:35,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:38,271 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 01:59:39,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:59:40,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:45,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:45,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:50,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 01:59:50,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 01:59:54,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 01:59:55,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:00,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:00,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:05,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:05,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:08,272 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:00:09,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:00:11,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:15,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:16,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:17,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:00:17,804 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:00:17,804 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:00:17,805 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:00:18,138 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:00:20,139 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:00:20,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:21,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:24,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:00:26,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:30,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:31,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:35,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:36,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:38,273 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:00:39,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:00:42,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:45,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:47,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:50,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:00:52,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:00:54,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:00:57,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:00,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:02,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:05,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:07,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:08,274 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:01:09,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:01:13,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:15,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:18,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:20,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:01:20,700 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:01:20,701 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:01:20,702 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:01:20,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:21,173 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:01:22,174 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:01:23,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:24,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:01:28,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:30,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:33,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:35,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:38,275 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:01:39,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:39,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:01:44,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:45,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:49,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:50,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:01:54,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:01:54,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:01:59,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:00,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:04,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:05,839 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:08,276 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:02:09,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:02:10,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:15,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:15,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:20,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:20,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:23,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:02:23,285 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:02:23,285 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:02:23,286 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:02:24,210 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:02:24,210 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:02:24,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:02:25,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:30,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:30,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:35,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:35,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:38,277 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:02:39,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:02:41,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:45,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:46,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:50,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:02:51,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:02:54,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:02:56,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:00,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:01,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:05,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:06,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:08,278 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:03:09,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:03:12,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:14,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:17,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:19,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:22,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:24,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:03:26,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:03:26,035 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:03:26,035 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:03:26,036 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:03:26,245 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:03:28,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:28,246 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:03:29,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:33,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:34,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:38,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:38,278 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:03:39,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:03:43,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:44,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:48,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:49,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:03:53,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:54,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:03:58,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:03:59,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:03,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:04,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:08,279 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:04:09,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:09,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:04:14,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:14,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:19,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:19,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:24,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:24,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:04:28,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:04:28,273 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:04:28,273 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:04:28,274 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:04:28,282 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:04:29,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:29,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:30,283 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:04:34,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:34,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:38,280 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:04:39,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:04:40,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:44,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:45,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:49,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:04:50,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:54,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:04:55,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:04:59,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:00,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:04,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:05,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:08,281 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:05:09,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:05:11,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:14,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:16,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:19,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:21,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:24,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:05:26,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:29,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:30,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:05:30,755 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:05:30,755 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:05:30,757 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:05:31,331 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:05:31,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:32,331 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:05:34,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:36,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:38,282 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:05:39,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:05:42,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:44,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:47,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:49,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:05:52,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:54,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:05:57,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:05:59,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:02,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:04,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:07,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:08,283 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:06:09,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:06:13,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:14,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:18,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:19,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:23,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:24,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:06:28,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:29,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:33,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:06:33,128 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:06:33,128 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:06:33,129 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:06:33,366 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:06:33,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:33,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:06:34,366 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:06:34,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:38,284 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:06:39,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:39,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:06:44,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:44,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:49,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:50,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:06:54,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:06:54,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:06:59,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:00,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:04,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:05,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:08,285 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:07:09,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:07:10,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:15,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:15,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:20,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:20,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:24,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:07:25,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:30,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:30,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:35,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:35,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:36,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:07:36,312 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:07:36,313 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:07:36,314 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:07:36,402 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:07:38,286 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:07:38,403 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:07:39,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:07:41,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:45,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:46,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:50,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:07:51,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:07:54,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:07:56,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:00,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:01,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:05,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:06,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:08,287 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:08:09,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:08:12,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:15,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:17,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:20,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:22,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:24,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:08:27,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:30,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:32,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:35,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:37,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:38,287 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:08:38,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:08:38,527 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:08:38,527 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:08:38,529 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:08:39,439 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:08:39,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:08:40,440 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:08:42,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:45,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:47,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:50,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:08:52,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:08:54,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:08:57,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:00,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:02,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:05,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:07,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:08,288 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:09:09,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:09:13,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:15,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:18,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:20,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:23,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:24,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:09:28,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:30,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:33,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:35,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:38,289 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:09:39,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:39,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:09:40,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:09:40,609 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:09:40,609 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:09:40,611 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:09:41,475 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:09:42,475 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:09:44,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:45,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:49,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:50,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:09:54,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:09:54,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:09:59,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:00,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:04,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:05,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:08,293 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:10:09,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:10:10,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:15,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:15,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:20,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:20,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:24,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:10:25,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:30,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:30,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:35,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:35,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:38,291 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:10:39,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:10:41,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:42,844 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:10:42,846 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:10:42,846 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:10:42,848 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:10:43,510 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:10:44,510 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:10:45,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:46,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:50,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:10:51,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:10:54,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:10:56,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:00,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:01,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:05,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:06,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:08,292 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:11:09,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:11:12,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:15,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:17,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:20,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:22,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:24,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:11:27,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:30,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:32,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:35,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:37,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:38,294 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:11:39,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:11:43,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:45,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:11:45,043 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:11:45,043 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:11:45,044 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:11:45,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:45,544 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:11:46,545 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:11:49,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:50,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:11:54,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:11:54,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:11:59,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:00,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:04,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:05,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:08,294 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:12:09,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:09,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:12:14,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:15,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:19,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:20,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:24,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:24,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:12:29,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:30,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:34,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:35,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:38,295 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:12:39,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:12:40,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:45,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:45,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:47,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:12:47,775 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:12:47,776 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:12:47,778 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:12:48,579 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:12:48,579 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:12:50,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:12:50,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:12:54,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:12:55,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:00,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:00,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:05,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:05,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:08,296 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:13:09,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:13:11,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:15,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:16,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:20,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:21,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:24,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:13:26,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:30,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:31,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:35,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:36,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:38,296 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:13:39,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:13:42,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:45,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:47,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:50,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:13:50,020 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:13:50,021 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:13:50,022 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:13:50,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:13:50,613 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:13:50,614 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:13:53,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:13:54,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:13:58,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:00,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:03,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:05,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:08,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:08,297 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:14:09,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:14:13,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:15,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:18,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:20,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:23,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:24,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:14:28,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:30,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:33,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:35,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:38,298 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:14:39,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:39,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:14:44,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:45,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:49,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:50,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:14:52,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:14:52,578 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:14:52,578 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:14:52,580 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:14:52,649 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:14:54,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:14:54,650 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:14:54,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:14:59,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:00,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:04,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:05,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:08,299 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:15:09,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:15:10,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:15,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:15,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:20,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:20,481 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:24,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:15:25,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:30,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:30,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:35,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:35,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:38,300 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:15:39,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:15:41,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:45,485 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:46,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:50,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:15:51,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:15:54,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:15:55,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:15:55,115 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:15:55,115 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:15:55,116 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:15:55,690 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:15:56,690 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:15:57,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:00,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:02,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:05,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:07,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:08,301 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:16:09,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:16:12,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:15,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:17,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:20,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:22,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:24,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:16:27,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:30,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:32,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:35,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:37,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:38,301 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:16:39,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:16:43,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:45,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:48,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:50,496 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:16:53,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:54,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:16:57,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:16:57,677 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:16:57,677 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:16:57,678 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:16:57,732 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:16:58,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:16:58,733 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:17:00,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:03,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:05,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:08,302 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:17:09,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:09,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:17:14,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:15,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:19,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:20,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:24,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:24,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:17:29,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:30,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:34,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:35,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:38,303 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:17:39,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:17:40,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:45,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:45,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:50,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:17:50,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:17:54,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:17:55,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:00,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:18:00,084 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:18:00,084 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:18:00,086 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:18:00,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:00,776 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:18:00,776 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:18:01,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:05,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:06,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:08,304 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:18:09,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:18:11,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:15,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:16,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:20,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:21,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:24,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:18:26,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:30,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:31,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:35,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:36,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:38,305 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:18:39,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:18:42,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:45,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:47,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:50,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:18:52,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:18:54,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:18:57,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:00,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:02,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:19:02,057 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:19:02,057 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:19:02,059 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:19:02,821 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:19:02,822 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:19:03,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:05,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:08,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:08,306 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:19:09,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:19:13,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:15,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:18,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:20,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:23,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:24,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:19:28,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:30,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:33,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:35,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:38,307 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:19:39,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:39,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:19:44,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:45,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:49,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:50,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:19:54,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:19:54,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:19:59,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:00,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:04,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:04,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:20:04,976 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:20:04,976 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:20:04,978 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:20:05,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:05,871 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:20:06,872 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:20:08,307 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:20:09,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:20:10,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:15,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:15,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:20,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:20,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:24,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:20:25,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:30,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:30,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:35,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:35,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:38,308 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:20:39,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:20:41,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:45,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:46,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:50,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:20:51,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:20:54,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:20:56,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:00,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:01,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:05,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:06,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:07,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:21:07,089 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:21:07,090 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:21:07,091 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:21:07,953 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:21:08,309 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:21:08,954 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:21:09,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:21:12,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:15,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:17,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:20,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:22,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:24,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:21:27,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:30,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:32,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:35,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:37,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:38,310 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:21:39,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:21:43,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:45,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:48,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:50,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:21:53,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:21:54,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:21:58,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:00,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:03,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:05,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:08,311 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:22:09,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:09,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:22:09,607 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:22:09,607 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:22:09,610 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:22:09,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:22:10,020 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:22:11,022 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:22:14,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:15,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:19,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:20,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:24,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:24,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:22:29,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:30,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:34,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:35,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:38,312 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:22:39,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:22:40,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:45,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:45,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:50,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:22:50,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:22:54,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:22:55,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:00,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:00,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:05,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:05,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:08,313 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:23:09,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:23:11,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:11,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:23:12,001 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:23:12,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:23:12,003 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:23:12,118 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:23:13,118 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:23:15,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:17,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:20,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:22,044 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:24,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:23:27,044 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:30,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:32,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:35,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:37,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:38,314 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:23:39,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:23:42,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:45,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:47,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:50,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:23:52,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:23:54,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:23:57,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:00,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:02,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:05,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:07,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:08,315 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:24:09,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:24:13,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:14,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:24:14,266 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:24:14,266 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:24:14,267 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:24:15,191 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:24:15,192 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:24:15,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:19,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:20,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:24,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:24,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:24:29,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:30,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:34,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:35,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:38,316 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:24:39,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:39,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:24:44,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:45,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:49,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:50,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:24:54,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:24:54,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:24:59,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:00,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:04,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:05,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:08,317 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:25:09,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:25:10,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:15,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:15,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:17,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:25:17,115 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:25:17,116 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:25:17,117 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:25:17,226 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:25:19,228 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:25:20,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:21,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:24,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:25:26,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:30,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:31,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:35,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:36,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:38,317 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:25:39,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:25:41,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:44,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:46,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:49,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:25:51,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:54,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:25:56,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:25:59,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:01,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:04,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:06,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:08,318 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:26:09,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:26:12,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:14,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:17,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:19,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:26:19,704 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:26:19,705 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:26:19,706 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:26:19,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:20,261 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:26:21,261 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:26:22,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:24,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:26:27,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:29,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:32,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:34,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:37,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:38,319 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:26:39,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:26:43,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:44,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:48,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:49,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:26:53,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:26:54,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:26:58,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:00,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:03,323 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:05,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:08,320 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:27:09,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:09,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:27:14,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:15,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:19,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:20,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:22,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:27:22,296 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:27:22,296 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:27:22,298 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:27:22,298 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:27:23,299 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:27:24,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:25,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:27:29,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:31,047 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:33,313 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:27:35,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:36,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:38,321 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:27:39,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:27:41,054 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:41,318 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:27:45,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:46,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:50,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:27:51,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:54,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:27:55,334 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:27:56,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:27:56,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:27:57,335 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:28:00,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:01,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:05,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:06,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:08,322 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:28:09,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:28:11,323 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:15,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:16,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:20,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:21,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:24,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:28:26,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:30,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:31,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:35,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:36,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:38,323 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:28:39,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:28:42,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:45,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:47,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:50,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:28:52,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:54,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:28:57,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:28:59,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:28:59,623 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:28:59,624 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:28:59,625 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:29:00,377 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:29:00,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:01,377 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:29:02,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:05,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:07,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:08,324 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:29:09,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:29:13,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:14,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:18,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:19,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:23,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:24,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:29:28,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:29,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:33,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:34,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:38,325 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:29:39,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:39,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:29:44,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:45,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:49,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:50,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:29:54,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:29:54,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:29:59,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:00,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:02,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:30:02,588 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:30:02,588 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:30:02,589 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:30:03,411 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:30:03,411 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:30:04,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:05,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:08,325 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:30:09,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:30:10,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:15,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:15,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:20,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:20,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:24,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:30:25,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:30,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:30,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:35,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:35,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:38,326 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:30:39,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:30:41,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:45,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:46,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:50,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:30:51,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:30:54,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:30:56,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:00,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:01,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:05,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:05,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:31:05,297 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:31:05,297 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:31:05,299 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:31:05,445 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:31:05,446 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:31:07,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:08,327 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:31:09,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:31:12,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:15,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:17,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:20,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:22,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:24,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:31:27,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:30,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:32,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:35,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:37,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:38,328 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:31:39,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:31:43,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:45,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:48,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:50,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:31:53,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:31:54,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:31:58,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:00,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:03,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:05,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:08,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:32:08,073 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:32:08,073 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:32:08,074 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:32:08,329 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:32:08,481 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:32:09,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:09,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:32:10,482 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:32:14,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:15,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:19,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:20,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:24,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:24,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:32:29,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:30,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:34,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:35,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:38,332 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:32:39,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:39,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:32:44,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:45,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:49,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:50,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:32:54,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:32:54,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:32:59,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:00,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:04,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:05,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:08,331 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:33:09,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:33:10,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:10,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:33:10,617 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:33:10,618 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:33:10,620 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:33:11,516 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:33:12,517 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:33:15,044 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:15,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:20,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:20,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:24,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:33:25,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:30,046 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:30,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:35,047 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:35,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:38,332 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:33:39,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:33:41,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:45,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:46,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:50,050 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:33:51,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:33:54,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:33:56,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:00,051 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:01,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:05,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:06,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:08,334 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:34:09,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:34:12,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:13,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:34:13,126 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:34:13,126 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:34:13,127 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:34:13,551 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:34:14,552 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:34:15,053 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:18,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:20,054 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:23,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:24,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:34:28,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:30,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:33,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:35,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:38,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:38,334 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:34:39,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:34:43,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:45,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:48,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:50,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:34:53,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:34:54,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:34:58,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:00,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:03,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:05,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:08,335 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:35:09,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:09,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:35:14,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:15,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:15,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:35:15,441 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:35:15,442 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:35:15,443 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:35:15,586 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:35:16,586 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:35:19,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:20,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:24,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:24,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:35:29,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:30,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:34,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:35,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:38,335 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:35:39,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:35:40,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:45,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:45,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:50,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:35:50,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:35:54,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:35:55,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:00,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:00,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:05,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:05,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:08,336 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:36:09,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:36:11,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:15,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:16,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:17,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:36:17,999 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:36:17,999 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:36:18,001 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:36:18,671 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:36:20,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:20,672 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:36:22,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:24,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:36:27,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:30,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:32,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:35,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:37,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:38,337 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:36:39,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:36:42,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:45,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:47,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:50,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:36:52,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:36:54,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:36:57,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:00,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:02,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:05,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:07,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:08,338 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:37:09,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:37:13,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:15,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:18,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:20,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:21,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:37:21,360 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:37:21,360 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:37:21,361 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:37:21,836 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:37:21,836 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:37:23,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:24,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:37:28,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:30,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:33,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:35,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:38,339 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:37:39,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:39,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:37:44,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:45,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:49,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:50,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:37:54,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:37:54,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:37:59,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:00,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:04,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:05,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:08,340 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:38:09,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:38:10,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:15,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:15,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:20,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:20,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:23,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:38:23,659 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:38:23,659 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:38:23,663 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:38:23,872 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:38:24,872 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:38:24,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:38:25,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:30,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:30,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:35,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:35,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:38,341 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:38:39,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:38:41,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:45,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:46,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:50,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:38:51,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:38:54,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:38:56,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:00,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:01,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:05,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:06,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:08,342 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:39:09,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:39:12,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:15,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:17,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:20,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:22,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:24,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:39:26,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:39:26,180 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:39:26,181 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:39:26,182 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:39:26,907 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:39:28,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:28,908 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:39:30,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:33,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:35,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:38,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:38,343 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:39:39,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:39:43,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:45,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:48,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:50,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:39:53,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:39:54,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:39:58,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:00,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:03,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:05,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:08,346 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:40:08,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:09,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:40:13,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:15,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:18,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:20,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:23,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:24,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:40:28,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:28,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:40:28,926 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:40:28,926 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:40:28,927 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:40:28,942 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:40:30,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:30,943 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:40:33,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:35,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:38,345 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:40:39,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:39,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:40:44,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:45,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:49,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:50,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:40:54,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:40:54,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:40:59,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:00,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:04,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:05,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:08,347 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:41:09,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:41:10,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:15,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:15,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:20,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:20,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:24,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:41:25,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:30,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:30,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:31,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:41:31,271 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:41:31,271 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:41:31,273 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:41:31,978 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:41:32,978 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:41:35,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:36,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:38,346 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:41:39,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:41:41,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:45,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:46,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:50,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:41:51,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:41:54,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:41:56,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:00,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:01,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:05,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:06,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:08,347 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:42:09,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:42:12,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:15,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:17,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:20,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:22,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:24,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:42:27,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:30,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:32,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:33,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:42:33,791 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:42:33,792 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:42:33,793 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:42:34,012 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:42:35,012 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:42:35,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:37,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:38,348 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:42:39,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:42:43,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:45,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:48,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:50,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:42:53,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:42:54,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:42:58,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:00,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:03,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:05,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:08,349 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:43:09,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:09,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:43:14,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:15,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:19,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:20,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:24,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:24,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:43:29,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:30,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:34,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:35,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:36,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:43:36,259 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:43:36,259 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:43:36,261 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:43:37,047 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:43:38,350 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:43:39,048 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:43:39,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:43:40,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:45,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:45,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:50,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:43:50,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:43:54,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:43:55,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:00,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:00,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:05,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:05,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:08,351 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:44:09,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:44:11,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:15,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:16,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:20,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:21,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:24,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:44:26,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:30,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:31,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:35,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:36,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:38,352 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:44:38,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:44:38,856 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:44:38,857 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:44:38,858 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:44:39,081 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:44:39,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:44:41,083 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:44:41,859 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:45,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:46,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:50,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:44:51,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:44:54,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:44:56,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:00,522 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:01,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:05,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:06,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:08,353 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:45:09,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:45:12,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:15,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:17,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:20,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:22,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:24,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:45:27,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:30,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:32,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:35,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:37,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:38,353 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:45:39,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:45:41,468 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:45:41,470 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:45:41,470 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:45:41,472 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:45:42,115 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:45:42,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:43,116 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:45:45,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:47,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:50,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:45:52,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:45:54,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:45:57,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:00,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:02,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:05,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:07,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:08,354 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:46:09,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:46:13,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:15,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:18,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:20,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:23,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:24,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:46:28,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:30,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:33,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:35,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:38,356 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:46:39,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:39,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:46:44,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:46:44,035 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:46:44,036 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:46:44,037 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:46:44,151 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:46:45,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:45,152 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:46:45,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:50,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:46:50,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:46:54,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:46:55,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:00,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:00,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:05,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:05,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:08,356 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:47:09,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:47:10,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:15,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:15,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:20,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:20,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:24,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:47:25,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:30,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:30,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:35,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:35,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:38,357 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:47:39,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:47:41,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:45,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:46,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:46,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:47:46,633 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:47:46,633 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:47:46,635 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:47:47,185 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:47:49,186 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:47:50,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:47:51,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:47:54,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:47:56,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:00,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:01,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:05,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:06,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:08,358 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:48:09,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:48:12,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:15,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:17,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:20,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:22,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:24,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:48:27,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:30,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:32,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:35,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:37,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:38,359 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:48:39,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:48:43,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:45,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:48,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:49,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:48:49,159 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:48:49,160 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:48:49,161 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:48:49,228 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:48:49,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:48:50,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:48:51,229 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:48:53,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:48:54,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:48:58,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:00,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:03,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:05,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:08,359 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:49:09,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:09,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:49:14,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:15,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:19,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:20,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:24,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:24,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:49:29,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:30,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:34,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:35,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:38,360 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:49:39,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:49:40,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:45,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:45,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:50,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:49:50,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:49:52,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:49:52,333 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:49:52,333 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:49:52,334 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:49:53,262 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:49:53,262 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:49:54,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:49:56,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:00,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:01,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:05,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:06,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:08,361 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:50:08,513 WARNING FileStreamThread:31085 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/iwakawa-koichi-q5-tohoku-nlp6723/trans_experiment/1g18vgup/file_stream. func: functools.partial(>, timeout=180.0), args: ('https://api.wandb.ai/files/iwakawa-koichi-q5-tohoku-nlp6723/trans_experiment/1g18vgup/file_stream',), kwargs: {'json': {'files': {'wandb-summary.json': {'offset': 0, 'content': ['{"training/loss": 2.2246241569519043, "training/perplexity": 9.250005600206721, "utils/batch_size": 1, "utils/global_batch_size": 256, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 256, "utils/iteration": 219, "optimizer/lr": 0.0001321011632680893, "optimizer/variance_l2": 0.023482480090775724, "optimizer/variance_sqrt_l2": 0.7321867298312661, "optimizer/momentum_l2": 0.11454680973332629, "optimizer/weight_l2": 1148.4176069705654, "optimizer/variance_l1": 0.5354156494140625, "optimizer/variance_sqrt_l1": 13664.0, "optimizer/momentum_l1": 2401.75, "optimizer/weight_l1": 52060160.0, "optimizer/variance_abs_max": 0.01214599609375, "optimizer/variance_sqrt_abs_max": 0.1103515625, "optimizer/momentum_abs_max": 0.00958251953125, "optimizer/weight_abs_max": 6.5, "stats/1_iteration_time": 62.592500790000486, "stats/tokens_per_sec": 4192.195497674059, "stats/tokens_per_sec_per_gpu": 4192.195497674059, "stats/tflops": 91.88828049055418, "_timestamp": 1730396866.6275494, "_runtime": 13989.305354356766, "_step": 219, "evaluation/val_loss": 2.1424567699432373, "evaluation/val_ppl": 8.520344734191895}']}, 'wandb-history.jsonl': {'offset': 218, 'content': ['{"training/loss": 2.2246241569519043, "training/perplexity": 9.250005600206721, "utils/batch_size": 1, "utils/global_batch_size": 256, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 256, "utils/iteration": 219, "optimizer/lr": 0.0001321011632680893, "optimizer/variance_l2": 0.023482480090775724, "optimizer/variance_sqrt_l2": 0.7321867298312661, "optimizer/momentum_l2": 0.11454680973332629, "optimizer/weight_l2": 1148.4176069705654, "optimizer/variance_l1": 0.5354156494140625, "optimizer/variance_sqrt_l1": 13664.0, "optimizer/momentum_l1": 2401.75, "optimizer/weight_l1": 52060160.0, "optimizer/variance_abs_max": 0.01214599609375, "optimizer/variance_sqrt_abs_max": 0.1103515625, "optimizer/momentum_abs_max": 0.00958251953125, "optimizer/weight_abs_max": 6.5, "stats/1_iteration_time": 62.592500790000486, "stats/tokens_per_sec": 4192.195497674059, "stats/tokens_per_sec_per_gpu": 4192.195497674059, "stats/tflops": 91.88828049055418, "_timestamp": 1730396866.6275494, "_runtime": 13989.305354356766, "_step": 219}']}, 'wandb-events.jsonl': {'offset': 467, 'content': ['{"system.proc.memory.availableMB": 51129.19, "system.memory": 11.63, "system.proc.memory.rssMB": 3030.26, "system.proc.memory.percent": 5.24, "system.network.sent": 91207737847.8, "system.network.recv": 127849670.73, "system.cpu": 5.57, "system.cpu.0.cpu_percent": 1.4, "system.cpu.1.cpu_percent": 11.33, "system.cpu.2.cpu_percent": 23.72, "system.cpu.3.cpu_percent": 40.63, "system.cpu.4.cpu_percent": 4.09, "system.cpu.5.cpu_percent": 0.63, "system.cpu.6.cpu_percent": 14.34, "system.cpu.7.cpu_percent": 0.0, "system.cpu.8.cpu_percent": 0.1, "system.cpu.9.cpu_percent": 0.0, "system.cpu.10.cpu_percent": 0.0, "system.cpu.11.cpu_percent": 0.03, "system.cpu.12.cpu_percent": 0.0, "system.cpu.13.cpu_percent": 0.0, "system.cpu.14.cpu_percent": 0.07, "system.cpu.15.cpu_percent": 6.52, "system.cpu.16.cpu_percent": 10.92, "system.cpu.17.cpu_percent": 16.31, "system.proc.cpu.threads": 60, "system.disk.\\\\.usagePercent": 0.0, "system.disk.\\\\.usageGB": 0.0, "system.disk.in": 5.85, "system.disk.out": 86611.55, "system.gpu.0.memoryAllocated": 98.58, "system.gpu.process.0.memoryAllocated": 98.58, "system.gpu.0.memoryAllocatedBytes": 42340052718.93, "system.gpu.process.0.memoryAllocatedBytes": 42340052718.93, "system.gpu.0.memory": 53.2, "system.gpu.process.0.memory": 53.2, "system.gpu.0.gpu": 95.33, "system.gpu.process.0.gpu": 95.33, "system.gpu.0.temp": 46.4, "system.gpu.process.0.temp": 46.4, "system.gpu.0.powerWatts": 347.41, "system.gpu.process.0.powerWatts": 347.41, "system.gpu.0.powerPercent": 86.85, "system.gpu.process.0.powerPercent": 86.85, "_wandb": true, "_timestamp": 1730396948.358639, "_runtime": 14071.036444}']}, 'output.log': {'offset': 817, 'content': ['2024-10-31T17:48:50.519055 ------------------------------------------------------------------\n', '2024-10-31T17:48:50.519055 iteration: 220 , TFLOPS: 91.985185056927, Tokens per sec: 4196.616549898384, Loss: 2.2362020015716553\n', '2024-10-31T17:48:50.519055 ------------------------------------------------------------------\n', '2024-10-31T17:48:50.519055 eval ppl=8.672445297241211, eval loss=2.1601507663726807\n']}}, 'dropped': 0}} +2024-11-01 02:50:09,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:50:11,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:15,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:16,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:20,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:21,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:24,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:50:26,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:30,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:31,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:35,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:36,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:38,362 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:50:39,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:50:42,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:45,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:47,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:50,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:50:52,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:50:54,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:50:54,837 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:50:54,838 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:50:54,840 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:50:54,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:50:55,309 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:50:57,310 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:50:57,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:00,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:02,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:05,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:07,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:08,363 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:51:09,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:51:13,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:15,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:18,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:20,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:23,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:24,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:51:28,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:30,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:33,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:35,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:38,365 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:51:39,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:39,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:51:44,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:45,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:49,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:50,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:51:54,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:51:54,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:51:57,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:51:57,774 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:51:57,775 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:51:57,776 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:51:58,350 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:51:59,351 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:51:59,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:00,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:04,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:05,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:08,365 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:52:09,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:52:10,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:15,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:15,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:20,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:20,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:24,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:52:25,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:30,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:30,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:35,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:35,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:38,366 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:52:39,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:52:41,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:45,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:46,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:50,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:52:51,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:54,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:52:56,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:52:59,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:00,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:53:00,103 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:53:00,104 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:53:00,105 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:53:00,384 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:53:01,384 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:53:02,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:04,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:07,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:08,367 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:53:09,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:53:12,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:14,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:17,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:19,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:22,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:24,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:53:27,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:29,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:32,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:34,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:37,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:38,367 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:53:39,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:53:43,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:44,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:48,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:49,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:53:53,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:54,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:53:58,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:53:59,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:02,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:54:02,941 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:54:02,941 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:54:02,942 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:54:03,420 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:54:03,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:04,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:05,422 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:54:08,368 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:54:09,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:09,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:54:14,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:14,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:19,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:19,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:24,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:24,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:54:29,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:29,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:34,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:34,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:38,369 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:54:39,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:54:40,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:44,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:45,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:49,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:54:50,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:54,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:54:55,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:54:59,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:00,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:04,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:05,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:05,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:55:05,695 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:55:05,696 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:55:05,697 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:55:06,456 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:55:07,457 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:55:08,371 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:55:09,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:55:11,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:14,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:16,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:19,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:21,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:24,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:55:26,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:29,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:31,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:34,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:36,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:38,371 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:55:39,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:55:42,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:44,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:47,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:49,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:55:52,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:54,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:55:57,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:55:59,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:02,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:05,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:07,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:08,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:56:08,126 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:56:08,126 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:56:08,128 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:56:08,372 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:56:08,490 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:56:09,491 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:56:09,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:56:13,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:15,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:18,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:20,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:23,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:24,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:56:28,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:30,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:33,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:35,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:38,373 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:56:39,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:39,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:56:44,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:45,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:49,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:50,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:56:54,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:56:54,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:56:59,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:00,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:04,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:05,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:08,374 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:57:09,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:57:10,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:10,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:57:10,494 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:57:10,494 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:57:10,495 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:57:10,524 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:57:11,525 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:57:15,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:15,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:20,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:20,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:24,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:57:25,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:30,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:30,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:35,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:35,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:38,374 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:57:39,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:57:41,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:45,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:46,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:50,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:57:51,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:57:54,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:57:56,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:00,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:01,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:05,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:06,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:08,375 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:58:09,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:58:12,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:12,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:58:12,873 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:58:12,873 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:58:12,874 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:58:13,560 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:58:13,560 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:58:15,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:17,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:20,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:22,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:24,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:58:27,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:30,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:32,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:35,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:37,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:38,376 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:58:39,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:58:43,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:45,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:48,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:50,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:58:53,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:58:54,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:58:58,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:00,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:03,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:05,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:08,377 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:59:09,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:09,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:59:14,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:15,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:15,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 02:59:15,800 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 02:59:15,800 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 02:59:15,802 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 02:59:16,598 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 02:59:17,598 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 02:59:19,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:20,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:24,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:24,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:59:29,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:30,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:34,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:35,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:38,378 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 02:59:39,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:59:40,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:45,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:45,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:50,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 02:59:50,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 02:59:54,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 02:59:55,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:00,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:00,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:05,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:05,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:08,379 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:00:09,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:00:11,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:15,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:16,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:17,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:00:17,949 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:00:17,950 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:00:17,951 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:00:18,632 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:00:19,632 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:00:20,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:21,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:24,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:00:26,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:30,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:31,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:35,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:36,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:38,379 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:00:39,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:00:42,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:45,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:47,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:50,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:00:52,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:00:54,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:00:57,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:00,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:02,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:05,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:07,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:08,380 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:01:09,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:01:13,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:15,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:18,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:20,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:20,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:01:20,613 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:01:20,613 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:01:20,615 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:01:20,666 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:01:21,666 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:01:23,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:24,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:01:28,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:30,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:33,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:35,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:38,381 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:01:39,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:39,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:01:44,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:45,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:49,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:50,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:01:54,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:01:54,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:01:59,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:00,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:04,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:05,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:08,381 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:02:09,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:02:10,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:15,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:15,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:20,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:20,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:22,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:02:22,914 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:02:22,915 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:02:22,917 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:02:23,701 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:02:23,701 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:02:24,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:02:25,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:30,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:30,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:35,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:35,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:38,382 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:02:39,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:02:41,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:45,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:46,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:50,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:02:51,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:02:54,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:02:56,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:00,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:01,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:05,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:06,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:08,386 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:03:09,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:03:11,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:15,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:16,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:20,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:21,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:24,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:03:25,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:03:25,433 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:03:25,434 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:03:25,435 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:03:25,734 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:03:26,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:27,735 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:03:30,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:31,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:35,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:36,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:38,384 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:03:39,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:03:42,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:45,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:47,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:50,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:03:52,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:03:54,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:03:57,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:00,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:02,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:05,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:07,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:08,386 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:04:09,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:04:13,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:15,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:18,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:20,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:23,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:24,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:04:27,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:04:27,979 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:04:27,979 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:04:27,980 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:04:28,779 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:04:29,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:29,780 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:04:30,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:34,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:35,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:38,386 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:04:39,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:39,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:04:44,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:45,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:49,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:50,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:04:54,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:04:54,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:04:59,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:00,515 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:04,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:05,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:08,390 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:05:09,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:09,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:05:14,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:15,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:19,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:20,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:24,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:24,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:05:29,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:30,481 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:05:30,483 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:05:30,483 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:05:30,484 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:05:30,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:30,852 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:05:31,853 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:05:34,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:35,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:38,390 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:05:39,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:05:40,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:45,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:45,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:50,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:05:50,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:05:54,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:05:55,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:00,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:00,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:05,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:05,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:08,392 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:06:09,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:06:11,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:15,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:16,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:20,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:21,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:24,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:06:26,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:30,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:31,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:32,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:06:32,902 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:06:32,902 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:06:32,904 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:06:33,887 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:06:33,888 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:06:35,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:36,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:38,392 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:06:39,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:06:42,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:45,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:47,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:50,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:06:52,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:06:54,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:06:57,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:00,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:02,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:05,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:07,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:08,393 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:07:09,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:07:13,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:15,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:18,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:20,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:23,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:24,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:07:28,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:30,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:33,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:35,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:07:35,573 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:07:35,574 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:07:35,576 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:07:35,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:35,922 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:07:37,923 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:07:38,394 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:07:39,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:39,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:07:44,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:45,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:49,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:50,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:07:54,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:07:54,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:07:59,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:00,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:04,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:05,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:08,395 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:08:09,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:08:10,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:15,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:15,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:20,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:20,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:24,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:08:25,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:30,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:30,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:35,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:35,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:37,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:08:37,808 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:08:37,808 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:08:37,810 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:08:37,957 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:08:38,396 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:08:39,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:08:39,959 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:08:41,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:46,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:46,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:51,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:08:51,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:08:54,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:08:56,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:00,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:01,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:05,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:06,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:08,397 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:09:09,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:09:12,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:15,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:17,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:20,050 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:22,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:24,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:09:27,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:30,051 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:32,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:35,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:37,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:38,398 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:09:39,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:09:40,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:09:40,265 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:09:40,265 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:09:40,267 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:09:40,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:09:40,992 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:09:41,993 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:09:42,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:45,054 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:47,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:50,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:09:52,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:09:54,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:09:57,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:00,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:02,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:05,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:07,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:08,398 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:10:09,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:10:13,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:15,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:18,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:20,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:23,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:24,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:10:28,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:30,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:33,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:35,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:38,401 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:10:38,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:39,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:10:43,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:10:43,095 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:10:43,095 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:10:43,096 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:10:44,027 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:10:44,027 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:10:44,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:45,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:49,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:50,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:10:54,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:10:54,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:10:59,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:00,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:04,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:05,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:08,400 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:11:09,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:09,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:11:14,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:15,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:19,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:20,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:24,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:24,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:11:29,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:30,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:34,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:35,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:38,402 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:11:39,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:11:40,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:45,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:45,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:45,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:11:45,639 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:11:45,640 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:11:45,641 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:11:46,061 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:11:48,062 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:11:50,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:11:50,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:11:54,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:11:55,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:00,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:00,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:05,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:05,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:08,402 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:12:09,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:12:11,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:15,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:16,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:20,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:21,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:24,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:12:26,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:30,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:31,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:35,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:36,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:38,403 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:12:39,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:12:42,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:45,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:47,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:47,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:12:47,972 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:12:47,972 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:12:47,974 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:12:48,095 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:12:50,096 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:12:50,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:12:52,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:12:54,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:12:57,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:00,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:02,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:05,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:07,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:08,404 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:13:09,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:13:13,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:15,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:18,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:20,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:23,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:24,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:13:28,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:30,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:33,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:35,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:38,405 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:13:39,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:39,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:13:44,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:45,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:49,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:50,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:13:50,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:13:50,442 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:13:50,443 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:13:50,444 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:13:51,130 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:13:52,131 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:13:54,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:13:54,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:13:59,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:00,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:04,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:05,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:08,406 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:14:09,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:14:10,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:15,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:15,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:20,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:20,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:24,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:14:25,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:30,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:30,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:35,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:35,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:38,406 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:14:39,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:14:41,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:45,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:46,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:50,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:14:51,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:14:53,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:14:53,205 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:14:53,206 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:14:53,207 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:14:54,165 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:14:54,166 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:14:54,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:14:57,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:00,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:02,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:05,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:07,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:08,407 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:15:09,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:15:12,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:15,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:17,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:20,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:22,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:24,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:15:27,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:30,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:32,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:35,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:37,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:38,408 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:15:39,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:15:43,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:45,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:48,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:50,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:15:53,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:15:54,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:15:55,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:15:55,540 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:15:55,540 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:15:55,541 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:15:56,201 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:15:56,201 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:15:58,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:00,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:03,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:05,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:08,409 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:16:09,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:09,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:16:14,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:15,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:19,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:20,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:24,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:24,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:16:29,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:30,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:34,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:35,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:38,410 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:16:39,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:16:40,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:45,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:45,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:50,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:50,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:16:54,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:16:55,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:16:57,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:16:57,730 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:16:57,730 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:16:57,731 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:16:58,237 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:17:00,238 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:17:00,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:00,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:05,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:05,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:08,410 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:17:09,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:17:11,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:15,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:16,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:20,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:21,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:24,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:17:26,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:30,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:31,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:35,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:36,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:38,411 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:17:39,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:17:42,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:45,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:47,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:50,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:17:52,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:17:54,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:17:57,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:00,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:00,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:18:00,896 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:18:00,897 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:18:00,898 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:18:01,298 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:18:02,299 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:18:02,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:05,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:07,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:08,415 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:18:09,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:18:13,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:15,588 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:18,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:20,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:23,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:24,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:18:28,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:30,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:33,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:35,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:38,414 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:18:39,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:39,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:18:44,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:45,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:49,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:50,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:18:54,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:18:54,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:18:59,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:00,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:03,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:19:03,299 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:19:03,300 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:19:03,302 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:19:03,332 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:19:05,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:05,333 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:19:05,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:08,414 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:19:09,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:19:10,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:15,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:15,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:20,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:20,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:25,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:19:25,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:30,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:30,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:35,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:35,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:38,415 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:19:40,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:19:41,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:45,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:46,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:50,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:19:51,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:19:55,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:19:56,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:00,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:01,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:05,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:05,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:20:05,779 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:20:05,780 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:20:05,781 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:20:06,369 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:20:07,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:07,369 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:20:08,415 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:20:09,371 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:20:10,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:20:12,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:15,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:18,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:19,376 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:20:20,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:23,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:24,379 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:20:25,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:20:28,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:30,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:33,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:35,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:38,416 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:20:38,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:39,391 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:20:40,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:20:41,392 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:20:43,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:45,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:48,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:50,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:20:53,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:20:55,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:20:58,464 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:00,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:03,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:05,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:08,419 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:21:09,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:10,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:21:14,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:15,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:19,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:20,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:24,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:25,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:21:29,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:30,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:34,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:35,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:38,420 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:21:40,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:21:40,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:42,845 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:21:42,847 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:21:42,848 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:21:42,849 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:21:43,431 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:21:45,433 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:21:45,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:45,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:50,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:21:50,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:21:55,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:21:55,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:00,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:00,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:05,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:05,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:08,421 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:22:10,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:22:11,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:15,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:16,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:20,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:21,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:25,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:22:26,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:30,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:31,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:35,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:36,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:38,422 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:22:40,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:22:42,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:44,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:22:44,913 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:22:44,913 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:22:44,915 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:22:45,467 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:22:45,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:47,468 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:22:47,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:50,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:22:52,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:22:55,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:22:57,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:00,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:02,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:05,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:07,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:08,423 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:23:10,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:23:13,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:15,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:18,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:20,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:23,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:25,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:23:28,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:30,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:33,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:35,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:38,424 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:23:39,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:40,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:23:44,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:45,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:47,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:23:47,465 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:23:47,465 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:23:47,467 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:23:47,503 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:23:49,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:49,504 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:23:50,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:23:54,468 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:23:55,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:23:59,468 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:00,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:04,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:05,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:08,424 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:24:10,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:24:10,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:15,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:15,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:20,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:20,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:25,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:24:25,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:30,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:30,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:35,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:35,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:38,426 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:24:40,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:24:41,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:45,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:46,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:49,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:24:49,960 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:24:49,960 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:24:49,962 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:24:50,539 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:24:50,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:24:51,540 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:24:51,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:24:55,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:24:56,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:00,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:01,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:05,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:06,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:08,429 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:25:10,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:25:12,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:15,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:17,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:20,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:22,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:25,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:25:27,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:30,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:32,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:35,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:37,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:38,427 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:25:40,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:25:43,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:45,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:48,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:50,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:25:52,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:25:52,559 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:25:52,560 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:25:52,562 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:25:52,575 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:25:53,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:25:55,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:25:55,577 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:25:58,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:00,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:03,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:05,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:08,428 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:26:09,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:10,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:26:14,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:15,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:19,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:20,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:24,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:25,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:26:29,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:30,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:34,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:35,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:38,430 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:26:40,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:26:40,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:45,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:45,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:50,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:50,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:26:55,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:26:55,054 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:26:55,054 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:26:55,056 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:26:55,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:26:55,610 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:26:56,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:26:57,611 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:27:00,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:01,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:05,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:06,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:08,430 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:27:10,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:27:11,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:15,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:16,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:20,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:21,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:25,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:27:26,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:30,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:31,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:35,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:36,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:38,431 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:27:40,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:27:42,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:45,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:47,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:50,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:27:52,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:55,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:27:57,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:27:57,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:27:57,739 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:27:57,739 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:27:57,740 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:27:58,644 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:27:59,645 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:28:00,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:02,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:05,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:07,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:08,432 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:28:10,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:28:13,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:15,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:18,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:20,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:23,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:25,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:28:28,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:30,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:33,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:35,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:38,433 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:28:39,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:40,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:28:44,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:45,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:49,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:50,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:28:54,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:28:55,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:28:59,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:00,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:29:00,651 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:29:00,651 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:29:00,652 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:29:00,677 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:29:00,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:03,678 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:29:04,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:05,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:08,433 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:29:10,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:10,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:15,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:15,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:20,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:20,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:25,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:25,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:30,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:30,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:35,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:35,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:38,434 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:29:40,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:41,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:45,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:46,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:50,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:29:51,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:29:55,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:29:56,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:00,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:01,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:03,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:30:03,574 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:30:03,574 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:30:03,576 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:30:03,711 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:30:05,712 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:30:05,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:06,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:08,435 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:30:10,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:12,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:15,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:17,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:20,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:22,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:25,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:27,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:30,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:32,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:35,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:37,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:38,436 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:30:40,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:43,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:45,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:48,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:50,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:30:53,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:30:55,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:30:58,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:00,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:03,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:05,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:31:05,734 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:31:05,734 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:31:05,736 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:31:05,744 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:31:05,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:06,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:31:07,745 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:31:08,437 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:31:09,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:10,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:14,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:15,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:19,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:20,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:24,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:25,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:29,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:30,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:34,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:35,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:38,438 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:31:40,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:40,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:45,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:45,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:50,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:31:50,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:31:55,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:31:55,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:00,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:00,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:05,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:05,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:08,442 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:32:08,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:32:08,452 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:32:08,452 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:32:08,454 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:32:08,778 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:32:09,778 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:32:10,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:10,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:15,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:15,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:20,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:20,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:25,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:25,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:30,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:30,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:35,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:35,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:38,442 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:32:40,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:41,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:45,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:46,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:50,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:32:51,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:32:55,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:32:56,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:00,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:01,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:05,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:06,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:08,447 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:33:10,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:11,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:33:11,158 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:33:11,159 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:33:11,160 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:33:11,812 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:33:12,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:13,813 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:33:15,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:17,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:20,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:22,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:25,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:27,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:30,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:32,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:35,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:37,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:38,444 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:33:40,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:42,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:45,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:47,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:50,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:33:52,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:33:55,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:33:57,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:00,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:02,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:05,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:07,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:08,445 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:34:10,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:13,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:13,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:34:13,576 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:34:13,576 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:34:13,578 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:34:13,845 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:34:15,846 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:34:15,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:18,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:20,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:23,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:25,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:28,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:30,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:33,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:35,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:38,446 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:34:39,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:40,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:44,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:45,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:49,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:50,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:34:54,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:34:55,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:34:59,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:00,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:04,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:05,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:08,446 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:35:10,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:10,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:15,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:15,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:16,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:35:16,202 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:35:16,203 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:35:16,204 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:35:16,879 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:35:17,879 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:35:20,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:21,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:25,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:26,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:30,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:31,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:35,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:36,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:38,447 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:35:40,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:41,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:45,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:46,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:50,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:35:51,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:35:55,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:35:56,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:00,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:01,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:05,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:06,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:08,448 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:36:10,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:12,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:15,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:17,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:19,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:36:19,189 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:36:19,189 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:36:19,191 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:36:19,914 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:36:20,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:21,915 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:36:23,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:25,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:28,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:30,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:33,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:35,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:38,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:38,449 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:36:40,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:43,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:45,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:48,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:50,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:36:53,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:36:55,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:36:58,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:00,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:03,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:05,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:08,450 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:37:09,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:10,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:37:14,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:15,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:19,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:20,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:21,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:37:21,302 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:37:21,302 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:37:21,304 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:37:21,948 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:37:23,949 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:37:25,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:37:25,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:30,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:30,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:35,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:35,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:38,450 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:37:40,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:37:40,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:45,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:45,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:50,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:37:50,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:37:55,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:37:55,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:00,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:00,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:05,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:05,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:08,451 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:38:10,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:38:11,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:15,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:16,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:20,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:21,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:24,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:38:24,111 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:38:24,111 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:38:24,113 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:38:24,982 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:38:25,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:38:25,983 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:38:27,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:30,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:32,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:35,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:37,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:38,452 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:38:40,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:38:42,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:45,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:47,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:50,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:38:52,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:38:55,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:38:57,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:00,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:02,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:05,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:07,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:08,453 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:39:10,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:39:13,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:16,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:18,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:21,048 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:23,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:25,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:39:26,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:39:26,630 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:39:26,631 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:39:26,632 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:39:27,017 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:39:28,018 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:39:28,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:31,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:33,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:36,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:38,454 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:39:39,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:40,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:39:44,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:46,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:49,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:51,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:39:54,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:39:55,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:39:59,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:01,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:04,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:06,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:08,455 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:40:10,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:40:10,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:15,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:16,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:20,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:21,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:25,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:40:25,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:28,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:40:28,913 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:40:28,913 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:40:28,914 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:40:29,052 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:40:30,052 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:40:30,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:31,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:35,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:36,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:38,456 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:40:40,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:40:41,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:46,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:46,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:51,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:40:51,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:40:55,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:40:56,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:01,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:01,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:06,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:06,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:08,457 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:41:10,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:41:12,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:16,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:17,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:21,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:22,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:25,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:41:27,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:31,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:31,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:41:31,390 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:41:31,391 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:41:31,392 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:41:32,100 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:41:33,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:34,101 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:41:36,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:38,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:38,457 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:41:40,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:41:43,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:46,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:48,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:51,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:41:53,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:41:55,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:41:58,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:01,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:03,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:06,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:08,459 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:42:09,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:10,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:42:14,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:16,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:19,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:21,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:24,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:25,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:42:29,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:31,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:33,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:42:33,680 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:42:33,681 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:42:33,682 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:42:34,134 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:42:34,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:36,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:36,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:42:38,459 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:42:40,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:42:40,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:45,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:46,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:50,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:42:51,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:42:55,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:42:55,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:00,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:01,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:05,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:06,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:08,486 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:43:10,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:43:10,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:15,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:16,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:20,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:21,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:25,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:43:25,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:30,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:31,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:35,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:36,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:36,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:43:36,552 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:43:36,552 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:43:36,554 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:43:37,169 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:43:38,169 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:43:38,486 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:43:40,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:43:41,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:46,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:46,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:51,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:43:51,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:43:55,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:43:56,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:01,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:01,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:06,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:06,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:08,487 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:44:10,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:44:12,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:16,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:17,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:21,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:22,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:25,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:44:27,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:30,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:32,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:35,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:37,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:38,488 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:44:39,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:44:39,022 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:44:39,022 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:44:39,023 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:44:39,203 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:44:40,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:44:40,203 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:44:43,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:45,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:48,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:50,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:44:53,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:44:55,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:44:58,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:00,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:03,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:05,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:08,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:08,489 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:45:10,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:13,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:15,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:18,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:20,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:23,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:25,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:28,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:30,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:33,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:35,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:38,490 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:45:39,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:40,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:41,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:45:41,221 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:45:41,221 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:45:41,222 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:45:41,237 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:45:42,238 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:45:45,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:45,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:50,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:45:50,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:55,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:55,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:00,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:00,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:05,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:05,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:08,491 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:46:10,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:10,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:15,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:15,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:20,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:20,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:25,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:25,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:30,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:30,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:35,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:35,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:38,492 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:46:40,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:41,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:44,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:46:44,037 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:46:44,037 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:46:44,038 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:46:44,272 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:46:45,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:46,273 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:46:47,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:50,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:46:52,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:55,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:57,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:00,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:02,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:05,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:07,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:08,493 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:47:10,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:12,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:15,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:17,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:20,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:22,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:25,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:27,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:30,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:32,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:35,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:37,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:38,494 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:47:40,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:43,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:45,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:46,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:47:46,812 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:47:46,812 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:47:46,814 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:47:47,307 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:47:48,308 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:47:48,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:50,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:47:53,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:55,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:58,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:00,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:03,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:05,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:08,497 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:48:09,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:10,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:14,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:15,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:19,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:20,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:24,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:25,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:29,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:30,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:34,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:35,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:38,496 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:48:40,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:40,496 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:45,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:45,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:49,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:48:49,566 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:48:49,567 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:48:49,569 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:50,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:48:50,345 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:48:50,345 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:48:50,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:55,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:55,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:00,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:00,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:05,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:05,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:08,497 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:49:10,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:49:11,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:15,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:16,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:20,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:21,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:25,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:49:26,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:30,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:31,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:35,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:36,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:38,498 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:49:40,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:49:42,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:45,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:47,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:50,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:49:51,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:49:51,818 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:49:51,818 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:49:51,820 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:49:52,383 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:49:52,383 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:49:52,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:55,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:49:57,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:00,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:02,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:05,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:07,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:08,499 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:50:10,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:50:13,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:15,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:18,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:20,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:23,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:25,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:50:28,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:30,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:33,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:35,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:38,499 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:50:39,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:40,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:50:44,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:45,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:49,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:50,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:50:54,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:50:54,298 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:50:54,299 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:50:54,300 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:50:54,420 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:50:55,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:50:55,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:50:56,421 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:51:00,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:00,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:05,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:05,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:08,500 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:51:10,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:51:10,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:15,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:15,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:20,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:20,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:25,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:51:25,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:30,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:30,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:35,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:35,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:38,502 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:51:40,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:51:41,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:45,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:46,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:50,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:51:51,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:55,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:51:56,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:51:56,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:51:56,642 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:51:56,643 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:51:56,644 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:51:57,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:51:57,456 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:51:58,457 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:52:00,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:02,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:05,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:07,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:08,503 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:52:10,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:52:12,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:15,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:17,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:20,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:22,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:25,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:52:27,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:30,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:32,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:35,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:37,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:38,504 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:52:40,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:52:43,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:45,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:48,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:50,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:52:53,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:52:55,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:52:58,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:00,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:53:00,120 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:53:00,121 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:53:00,122 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:53:00,492 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:53:00,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:02,493 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:53:04,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:05,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:08,505 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:53:09,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:10,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:53:14,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:15,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:19,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:20,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:24,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:25,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:53:29,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:30,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:34,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:35,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:38,506 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:53:40,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:53:40,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:45,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:45,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:50,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:53:50,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:53:55,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:53:55,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:00,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:00,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:02,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:54:02,580 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:54:02,580 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:54:02,582 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:54:03,528 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:54:04,528 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:54:05,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:05,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:08,506 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:54:10,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:54:11,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:15,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:16,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:20,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:21,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:25,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:54:26,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:30,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:31,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:35,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:36,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:38,507 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:54:40,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:54:42,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:45,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:47,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:50,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:54:52,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:54:55,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:54:57,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:00,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:02,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:05,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:55:05,499 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:55:05,499 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:55:05,501 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:55:05,562 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:55:05,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:06,563 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:55:08,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:08,508 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:55:10,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:55:13,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:15,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:18,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:20,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:23,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:25,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:55:28,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:30,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:33,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:35,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:38,509 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:55:39,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:40,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:55:44,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:45,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:49,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:50,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:55:54,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:55:55,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:55:59,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:00,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:04,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:05,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:07,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:56:07,887 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:56:07,887 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:56:07,888 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:56:08,510 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:56:08,598 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:56:09,598 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:56:10,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:56:10,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:15,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:15,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:20,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:20,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:25,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:56:25,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:30,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:30,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:35,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:35,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:38,511 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:56:40,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:56:41,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:45,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:46,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:50,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:56:51,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:56:55,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:56:56,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:00,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:01,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:05,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:06,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:08,512 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:57:10,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:57:10,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:57:10,699 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:57:10,699 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:57:10,700 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:57:11,633 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:57:11,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:13,634 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:57:15,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:16,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:20,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:21,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:25,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:57:26,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:30,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:31,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:35,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:36,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:38,513 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:57:40,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:57:42,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:45,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:47,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:50,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:57:52,515 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:57:55,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:57:57,515 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:00,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:02,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:05,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:07,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:08,515 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:58:10,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:58:13,323 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:58:13,325 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:58:13,326 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:58:13,326 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:13,327 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:58:13,746 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:58:15,747 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:58:15,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:18,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:20,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:23,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:25,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:58:28,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:30,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:33,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:35,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:38,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:38,516 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:58:40,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:58:43,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:45,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:48,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:50,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:58:53,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:58:55,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:58:58,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:00,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:03,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:05,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:08,517 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:59:09,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:10,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:59:14,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:15,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:59:15,586 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 03:59:15,586 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:59:15,587 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:59:15,782 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 03:59:15,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:17,786 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 03:59:19,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:20,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:24,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:25,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:59:29,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:30,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:34,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:35,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:38,518 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 03:59:40,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:59:40,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:45,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:45,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:50,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:59:50,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 03:59:55,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:59:55,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:00,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:00,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:05,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:05,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:08,518 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:00:10,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:00:11,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:15,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:16,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:18,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:00:18,366 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:00:18,366 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:00:18,402 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:00:18,829 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:00:19,830 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:00:20,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:22,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:25,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:00:27,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:30,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:32,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:35,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:37,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:38,523 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:00:40,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:00:42,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:45,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:47,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:50,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:00:52,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:00:55,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:00:57,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:00,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:02,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:06,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:07,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:08,523 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:01:10,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:01:13,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:16,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:18,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:20,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:01:20,469 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:01:20,469 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:01:20,471 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:01:20,865 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:01:21,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:21,866 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:01:24,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:25,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:01:29,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:31,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:34,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:36,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:38,524 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:01:39,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:40,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:01:44,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:46,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:49,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:51,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:01:54,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:01:55,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:01:59,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:01,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:04,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:06,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:08,525 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:02:10,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:02:10,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:15,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:16,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:20,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:21,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:23,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:02:23,251 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:02:23,252 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:02:23,254 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:02:23,902 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:02:25,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:02:25,904 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:02:26,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:31,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:31,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:36,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:36,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:38,526 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:02:40,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:02:41,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:46,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:46,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:51,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:02:51,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:02:55,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:02:56,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:00,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:01,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:05,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:06,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:08,531 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:03:10,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:03:11,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:15,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:16,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:20,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:21,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:25,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:03:26,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:03:26,126 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:03:26,127 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:03:26,128 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:03:26,939 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:03:27,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:27,940 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:03:30,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:32,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:35,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:37,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:38,532 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:03:40,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:03:42,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:45,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:47,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:50,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:03:52,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:03:55,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:03:57,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:00,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:02,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:05,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:07,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:08,532 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:04:10,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:04:13,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:15,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:18,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:20,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:23,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:25,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:04:28,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:28,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:04:28,689 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:04:28,689 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:04:28,690 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:04:28,977 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:04:29,977 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:04:30,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:33,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:35,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:38,533 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:04:39,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:40,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:04:44,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:45,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:49,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:50,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:04:54,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:04:55,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:04:59,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:00,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:04,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:05,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:08,534 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:05:10,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:05:10,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:15,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:15,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:20,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:20,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:25,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:05:25,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:30,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:30,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:30,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:05:30,752 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:05:30,753 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:05:30,754 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:05:31,012 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:05:32,013 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:05:35,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:35,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:38,535 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:05:40,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:05:41,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:45,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:46,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:50,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:05:51,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:05:55,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:05:56,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:00,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:01,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:05,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:06,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:08,536 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:06:10,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:06:12,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:15,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:17,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:20,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:22,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:25,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:06:27,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:30,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:32,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:33,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:06:33,445 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:06:33,446 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:06:33,447 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:06:34,048 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:06:35,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:36,049 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:06:38,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:38,537 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:06:40,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:06:43,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:45,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:48,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:50,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:06:53,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:06:55,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:06:58,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:00,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:03,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:05,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:08,538 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:07:09,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:10,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:07:14,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:15,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:19,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:20,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:24,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:25,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:07:29,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:30,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:34,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:35,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:35,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:07:35,786 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:07:35,786 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:07:35,788 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:07:36,082 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:07:38,083 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:07:38,539 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:07:40,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:07:40,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:45,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:45,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:50,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:07:50,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:07:55,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:07:55,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:00,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:00,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:05,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:05,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:08,540 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:08:10,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:08:11,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:15,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:16,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:20,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:21,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:25,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:08:26,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:30,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:31,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:35,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:36,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:38,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:08:38,541 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:08:38,544 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:08:38,545 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:08:38,546 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:08:39,117 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:08:40,118 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:08:40,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:08:41,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:45,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:46,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:50,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:08:51,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:08:55,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:08:56,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:00,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:01,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:05,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:06,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:08,543 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:09:10,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:09:12,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:15,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:17,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:20,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:22,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:25,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:09:27,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:30,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:32,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:35,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:37,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:38,545 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:09:40,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:09:41,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:09:41,265 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:09:41,266 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:09:41,267 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:09:42,156 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:09:43,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:44,157 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:09:45,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:48,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:50,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:09:53,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:09:55,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:09:58,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:00,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:03,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:05,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:08,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:08,545 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:10:10,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:10:13,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:15,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:18,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:20,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:23,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:25,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:10:28,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:30,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:33,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:35,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:38,546 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:10:39,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:40,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:10:43,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:10:43,904 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:10:43,904 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:10:43,905 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:10:44,196 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:10:44,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:45,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:46,198 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:10:49,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:50,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:10:54,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:10:55,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:10:59,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:00,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:04,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:05,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:08,549 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:11:10,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:11:10,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:15,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:15,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:20,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:20,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:25,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:11:25,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:30,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:30,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:35,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:35,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:38,548 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:11:40,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:11:41,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:45,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:46,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:11:46,448 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:11:46,448 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:11:46,451 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:11:47,273 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:11:47,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:48,274 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:11:50,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:11:52,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:11:55,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:11:57,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:00,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:02,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:05,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:07,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:08,549 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:12:10,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:12:12,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:15,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:17,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:20,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:22,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:25,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:12:27,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:30,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:32,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:35,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:37,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:38,550 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:12:40,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:12:43,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:45,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:48,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:48,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:12:48,874 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:12:48,875 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:12:48,876 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:12:49,328 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:12:50,329 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:12:50,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:53,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:12:56,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:12:56,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:12:59,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:00,335 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:13:01,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:04,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:06,339 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:13:06,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:08,340 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:13:08,550 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:13:10,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:10,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:13:15,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:16,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:20,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:21,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:13:22,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:22,348 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:13:24,351 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:13:25,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:13:26,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:31,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:31,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:36,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:36,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:38,551 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:13:40,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:13:42,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:46,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:47,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:51,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:13:52,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:13:55,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:13:57,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:01,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:02,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:06,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:07,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:08,552 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:14:10,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:14:13,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:16,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:18,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:21,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:23,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:24,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:14:24,970 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:14:24,971 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:14:24,972 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:14:25,392 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:14:25,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:14:26,393 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:14:28,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:31,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:33,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:36,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:38,553 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:14:39,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:40,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:14:44,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:46,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:49,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:51,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:14:54,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:14:55,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:14:59,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:01,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:04,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:06,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:08,554 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:15:10,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:15:10,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:15,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:16,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:20,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:21,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:25,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:15:25,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:27,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:15:27,506 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:15:27,507 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:15:27,508 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:15:28,427 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:15:30,428 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:15:31,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:31,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:36,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:36,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:38,555 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:15:40,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:15:41,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:46,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:46,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:51,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:15:51,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:15:55,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:15:56,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:01,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:01,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:06,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:06,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:08,556 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:16:10,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:16:12,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:16,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:17,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:21,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:22,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:25,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:16:27,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:30,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:16:30,092 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:16:30,092 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:16:30,095 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:16:30,463 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:16:31,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:32,464 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:16:33,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:36,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:38,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:38,556 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:16:40,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:16:43,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:46,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:48,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:51,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:16:53,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:16:55,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:16:58,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:01,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:03,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:06,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:08,559 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:17:08,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:10,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:17:13,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:16,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:18,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:21,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:23,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:25,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:17:28,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:31,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:32,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:17:32,624 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:17:32,624 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:17:32,625 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:17:33,510 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:17:33,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:34,511 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:17:36,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:38,560 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:17:39,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:40,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:17:44,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:46,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:49,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:51,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:17:54,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:17:55,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:17:59,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:01,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:04,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:06,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:08,563 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:18:09,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:10,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:18:14,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:16,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:19,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:21,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:24,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:25,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:18:29,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:31,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:34,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:35,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:18:35,341 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:18:35,342 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:18:35,343 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:18:35,545 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:18:36,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:36,546 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:18:38,562 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:18:40,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:18:40,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:45,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:46,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:50,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:18:51,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:18:55,510 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:18:55,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:00,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:01,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:05,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:06,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:08,564 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:19:10,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:19:11,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:16,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:16,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:21,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:21,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:25,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:19:26,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:31,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:31,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:36,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:36,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:37,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:19:37,567 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:19:37,568 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:19:37,570 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:19:37,580 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:19:38,563 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:19:38,581 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:19:40,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:19:42,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:46,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:47,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:51,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:19:52,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:19:55,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:19:57,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:01,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:02,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:06,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:07,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:08,564 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:20:10,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:20:13,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:16,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:18,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:21,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:23,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:25,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:20:28,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:31,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:33,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:36,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:38,565 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:20:39,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:39,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:20:39,924 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:20:39,924 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:20:39,926 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:20:40,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:20:40,613 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:20:42,615 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:20:44,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:46,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:49,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:51,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:20:54,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:20:55,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:20:59,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:01,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:04,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:06,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:08,566 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:21:10,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:21:10,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:15,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:16,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:20,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:21,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:25,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:21:25,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:30,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:31,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:35,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:36,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:38,567 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:21:40,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:21:41,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:42,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:21:42,558 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:21:42,558 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:21:42,559 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:21:42,668 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:21:44,669 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:21:46,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:46,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:51,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:21:51,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:21:55,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:21:56,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:01,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:01,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:06,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:06,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:08,568 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:22:10,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:22:12,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:16,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:17,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:21,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:22,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:25,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:22:27,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:31,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:32,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:36,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:37,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:38,569 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:22:40,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:22:43,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:45,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:22:45,221 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:22:45,222 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:22:45,223 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:22:45,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:45,705 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:22:46,705 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:22:49,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:50,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:22:54,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:22:55,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:22:59,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:00,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:04,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:05,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:08,570 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:23:09,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:10,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:23:14,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:15,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:19,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:20,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:24,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:25,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:23:29,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:30,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:34,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:35,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:38,571 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:23:40,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:23:40,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:45,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:45,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:47,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:23:47,997 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:23:47,998 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:23:47,999 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:23:48,749 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:23:50,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:23:50,750 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:23:51,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:23:55,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:23:56,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:00,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:01,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:05,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:06,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:08,572 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:24:10,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:24:11,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:15,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:16,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:20,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:21,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:25,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:24:26,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:30,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:31,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:35,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:36,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:38,572 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:24:40,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:24:42,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:45,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:47,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:50,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:24:50,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:24:50,751 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:24:50,752 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:24:50,753 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:24:50,781 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:24:52,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:24:52,782 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:24:55,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:24:57,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:00,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:02,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:05,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:07,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:08,573 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:25:10,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:25:13,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:15,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:18,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:20,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:23,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:25,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:25:28,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:30,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:33,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:35,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:38,574 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:25:39,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:40,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:25:44,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:45,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:49,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:50,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:25:53,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:25:53,242 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:25:53,242 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:25:53,243 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:25:53,816 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:25:54,816 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:25:55,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:25:55,522 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:26:00,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:00,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:05,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:05,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:08,575 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:26:10,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:26:10,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:15,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:15,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:20,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:20,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:25,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:26:25,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:30,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:30,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:35,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:35,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:38,576 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:26:40,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:26:41,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:45,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:46,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:50,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:26:51,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:26:55,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:26:56,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:26:56,066 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:26:56,067 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:26:56,068 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:26:56,852 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:26:56,852 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:26:57,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:00,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:02,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:05,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:07,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:08,577 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:27:10,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:27:12,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:15,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:17,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:20,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:22,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:25,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:27:27,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:30,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:32,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:35,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:37,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:38,578 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:27:40,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:27:43,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:45,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:48,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:50,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:27:53,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:27:55,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:27:58,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:27:58,504 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:27:58,505 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:27:58,506 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:27:58,886 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:27:59,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:00,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:00,887 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:28:04,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:05,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:08,578 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:28:09,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:10,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:28:14,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:15,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:19,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:20,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:24,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:25,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:28:29,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:30,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:34,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:35,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:38,579 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:28:40,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:28:40,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:45,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:45,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:50,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:28:50,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:28:55,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:28:55,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:00,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:00,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:00,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:29:00,854 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:29:00,855 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:29:00,857 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:29:00,920 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:29:02,921 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:29:05,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:05,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:08,580 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:29:10,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:29:11,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:15,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:16,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:20,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:21,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:25,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:29:26,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:30,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:31,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:35,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:36,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:38,581 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:29:40,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:29:42,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:45,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:47,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:50,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:29:52,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:29:55,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:29:57,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:00,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:02,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:03,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:30:03,701 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:30:03,702 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:30:03,703 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:30:03,955 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:30:04,956 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:30:05,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:07,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:08,582 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:30:10,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:30:13,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:15,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:18,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:20,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:23,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:25,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:30:28,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:30,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:33,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:35,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:38,582 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:30:39,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:40,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:30:44,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:45,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:49,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:50,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:30:54,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:30:55,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:30:59,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:00,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:04,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:05,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:06,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:31:06,519 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:31:06,519 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:31:06,521 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:31:06,990 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:31:08,583 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:31:08,991 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:31:10,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:31:10,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:15,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:15,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:20,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:20,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:25,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:31:25,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:30,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:30,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:35,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:35,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:38,584 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:31:40,518 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:31:41,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:45,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:46,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:50,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:31:51,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:31:55,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:31:56,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:00,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:01,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:05,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:06,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:08,585 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:32:08,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:32:08,713 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:32:08,713 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:32:08,715 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:32:09,024 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:32:10,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:32:11,026 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:32:11,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:15,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:16,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:20,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:21,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:25,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:32:26,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:30,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:31,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:35,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:36,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:38,586 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:32:40,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:32:42,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:45,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:47,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:50,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:32:52,588 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:32:55,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:32:57,588 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:01,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:02,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:06,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:07,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:08,587 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:33:10,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:33:11,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:33:11,141 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:33:11,141 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:33:11,142 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:33:12,060 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:33:13,060 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:33:13,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:16,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:18,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:21,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:23,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:25,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:33:28,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:31,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:33,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:36,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:38,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:38,588 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:33:40,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:33:43,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:46,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:48,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:51,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:33:53,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:33:55,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:33:58,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:01,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:03,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:06,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:08,589 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:34:09,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:10,519 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:34:13,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:34:13,732 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:34:13,732 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:34:13,733 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:34:14,093 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:34:14,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:34:15,094 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:34:15,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:16,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:20,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:21,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:25,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:25,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:34:30,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:31,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:35,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:36,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:38,589 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:34:40,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:34:40,590 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:45,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:46,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:50,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:34:51,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:34:55,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:34:55,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:00,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:01,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:05,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:06,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:08,590 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:35:10,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:35:11,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:16,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:16,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:17,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:35:17,534 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:35:17,535 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:35:17,543 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:35:18,129 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:35:19,130 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:35:21,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:22,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:25,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:35:27,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:31,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:32,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:36,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:37,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:38,591 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:35:40,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:35:42,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:46,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:47,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:51,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:35:52,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:35:55,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:35:57,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:01,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:02,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:06,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:07,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:08,592 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:36:10,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:36:13,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:16,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:18,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:20,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:36:20,076 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:36:20,077 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:36:20,078 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:36:20,164 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:36:21,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:21,165 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:36:24,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:25,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:36:29,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:31,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:34,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:36,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:38,593 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:36:39,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:40,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:36:44,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:46,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:49,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:51,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:36:54,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:36:55,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:36:59,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:01,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:04,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:06,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:08,594 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:37:10,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:37:10,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:15,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:16,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:20,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:21,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:22,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:37:22,670 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:37:22,671 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:37:22,672 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:37:23,199 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:37:24,199 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:37:25,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:37:25,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:30,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:31,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:35,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:36,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:38,595 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:37:40,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:37:41,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:46,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:46,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:51,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:37:51,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:37:55,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:37:56,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:01,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:01,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:06,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:06,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:08,595 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:38:10,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:38:12,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:16,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:17,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:21,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:22,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:25,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:38:25,222 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:38:25,222 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:38:25,223 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:38:25,233 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:38:25,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:38:28,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:28,235 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:38:31,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:33,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:36,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:38,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:38,596 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:38:40,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:38:43,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:46,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:48,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:51,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:38:53,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:38:55,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:38:58,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:01,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:03,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:06,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:08,597 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:39:09,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:10,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:39:14,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:16,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:19,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:21,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:24,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:25,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:39:27,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:39:27,627 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:39:27,627 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:39:27,628 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:39:28,268 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:39:29,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:30,282 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:39:31,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:34,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:36,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:38,598 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:39:40,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:40,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:39:45,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:46,142 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:50,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:51,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:39:55,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:39:55,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:40:00,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:01,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:05,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:06,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:08,599 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:40:10,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:40:11,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:16,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:16,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:21,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:21,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:25,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:40:26,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:29,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:40:29,822 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:40:29,822 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:40:29,823 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:40:30,314 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:40:31,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:31,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:32,315 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:40:36,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:36,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:38,600 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:40:40,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:40:42,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:46,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:47,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:51,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:40:52,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:40:55,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:40:57,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:01,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:02,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:06,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:07,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:08,603 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:41:10,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:41:13,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:16,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:18,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:21,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:23,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:25,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:41:28,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:31,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:32,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:41:32,141 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:41:32,142 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:41:32,143 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:41:32,346 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:41:34,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:34,347 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:41:36,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:38,601 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:41:39,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:40,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:41:44,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:46,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:49,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:51,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:41:54,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:41:55,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:41:59,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:01,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:04,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:06,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:08,603 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:42:10,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:10,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:42:15,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:16,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:20,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:21,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:25,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:42:25,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:30,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:31,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:34,840 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:42:34,842 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:42:34,843 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:42:34,844 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:42:35,380 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:42:35,844 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:36,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:36,380 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:42:38,603 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:42:40,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:42:41,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:46,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:46,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:51,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:42:51,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:42:55,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:42:56,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:01,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:01,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:06,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:06,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:08,604 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:43:10,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:43:12,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:16,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:17,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:21,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:22,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:25,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:43:27,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:31,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:32,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:36,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:37,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:43:37,275 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:43:37,276 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:43:37,277 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:43:37,414 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:43:38,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:38,605 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:43:40,416 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:43:40,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:43:43,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:46,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:48,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:51,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:43:53,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:43:55,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:43:58,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:01,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:03,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:06,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:08,606 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:44:09,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:10,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:44:14,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:16,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:19,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:21,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:24,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:25,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:44:29,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:31,460 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:34,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:36,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:38,607 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:44:39,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:44:39,802 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:44:39,802 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:44:39,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:39,804 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:44:40,448 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:44:40,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:44:42,449 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:44:44,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:46,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:49,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:51,464 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:44:54,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:44:55,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:44:59,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:01,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:04,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:06,466 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:08,608 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:45:10,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:45:10,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:15,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:16,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:20,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:21,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:25,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:45:25,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:30,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:31,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:35,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:36,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:38,609 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:45:40,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:45:41,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:42,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:45:42,166 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:45:42,167 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:45:42,168 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:45:42,482 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:45:44,483 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:45:46,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:47,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:51,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:45:52,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:45:55,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:45:57,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:01,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:02,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:06,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:07,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:08,609 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:46:10,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:46:12,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:16,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:17,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:21,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:22,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:25,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:46:27,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:31,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:32,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:36,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:37,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:38,610 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:46:40,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:46:43,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:44,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:46:44,475 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:46:44,475 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:46:44,477 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:46:44,518 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:46:46,519 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:46:46,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:49,515 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:51,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:46:54,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:46:55,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:46:59,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:01,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:04,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:06,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:08,611 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:47:09,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:10,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:47:14,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:15,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:19,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:20,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:24,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:25,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:47:29,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:30,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:34,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:35,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:38,612 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:47:40,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:47:40,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:45,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:45,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:47,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:47:47,299 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:47:47,299 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:47:47,301 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:47:47,552 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:47:48,553 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:47:50,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:47:51,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:47:55,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:47:56,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:00,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:01,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:05,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:06,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:08,612 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:48:10,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:48:11,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:15,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:16,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:20,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:21,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:25,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:48:26,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:30,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:31,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:35,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:36,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:38,613 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:48:40,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:48:42,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:45,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:47,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:50,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:48:50,142 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:48:50,143 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:48:50,144 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:48:50,587 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:48:50,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:48:52,589 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:48:53,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:48:55,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:48:58,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:00,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:03,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:05,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:08,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:08,614 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:49:10,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:49:13,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:15,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:18,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:20,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:23,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:25,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:49:28,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:30,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:33,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:35,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:38,615 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:49:39,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:40,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:49:44,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:45,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:49,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:50,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:49:52,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:49:52,741 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:49:52,741 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:49:52,746 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:49:53,623 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:49:54,623 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:49:54,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:49:55,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:49:59,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:00,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:04,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:05,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:08,616 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:50:10,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:50:10,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:15,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:15,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:20,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:20,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:25,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:50:25,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:30,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:30,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:35,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:35,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:38,617 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:50:40,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:50:41,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:45,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:46,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:50,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:50:51,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:50:55,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:50:55,270 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:50:55,271 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:50:55,273 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:50:55,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:50:55,658 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:50:56,658 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:50:57,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:00,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:02,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:05,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:07,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:08,618 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:51:10,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:51:12,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:15,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:17,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:20,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:22,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:25,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:51:27,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:30,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:32,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:35,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:37,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:38,619 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:51:40,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:51:43,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:46,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:48,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:51,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:51:53,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:51:55,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:51:57,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:51:57,958 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:51:57,958 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:51:57,959 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:51:58,694 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:51:58,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:00,695 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:52:01,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:03,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:06,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:08,620 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:52:09,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:10,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:52:14,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:16,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:19,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:21,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:24,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:25,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:52:29,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:31,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:34,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:36,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:38,621 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:52:40,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:52:40,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:45,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:46,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:50,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:52:51,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:52:55,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:52:55,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:00,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:00,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:53:00,649 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:53:00,649 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:53:00,650 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:53:00,729 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:53:01,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:02,730 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:53:05,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:06,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:08,622 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:53:10,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:53:11,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:16,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:16,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:21,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:21,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:25,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:53:26,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:31,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:31,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:36,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:36,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:38,623 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:53:40,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:53:42,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:46,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:47,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:51,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:53:52,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:53:55,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:53:57,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:01,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:02,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:02,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:54:02,989 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:54:02,990 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:54:02,991 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:54:03,764 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:54:04,764 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:54:06,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:07,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:08,623 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:54:10,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:54:13,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:16,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:18,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:21,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:23,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:25,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:54:28,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:31,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:33,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:36,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:38,624 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:54:39,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:40,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:54:44,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:46,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:49,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:51,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:54:54,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:54:55,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:54:59,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:01,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:04,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:05,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:55:05,704 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:55:05,704 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:55:05,706 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:55:05,797 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:55:06,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:06,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:55:06,798 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:55:08,625 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:55:08,799 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:55:10,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:55:10,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:15,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:16,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:20,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:21,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:25,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:55:25,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:30,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:31,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:35,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:36,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:38,626 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:55:40,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:55:41,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:46,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:46,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:51,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:55:51,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:55:55,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:55:56,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:01,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:01,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:06,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:06,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:08,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:56:08,593 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:56:08,593 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:56:08,594 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:56:08,627 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:56:08,831 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:56:10,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:56:10,833 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:56:12,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:16,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:17,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:21,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:22,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:25,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:56:27,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:31,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:32,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:36,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:37,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:38,628 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:56:40,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:56:43,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:46,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:48,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:51,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:56:53,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:56:55,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:56:58,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:01,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:03,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:06,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:08,629 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:57:09,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:10,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:57:11,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:57:11,252 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:57:11,252 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:57:11,253 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:57:11,867 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:57:12,867 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:57:15,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:16,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:20,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:21,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:25,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:25,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:57:30,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:31,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:35,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:36,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:38,631 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:57:40,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:57:40,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:45,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:46,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:50,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:57:51,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:57:55,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:57:55,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:00,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:01,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:05,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:06,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:08,632 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:58:10,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:58:11,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:13,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:58:13,160 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:58:13,160 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:58:13,161 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:58:13,901 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:58:14,901 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:58:16,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:17,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:21,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:22,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:25,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:58:27,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:31,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:32,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:36,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:37,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:38,633 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:58:40,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:58:42,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:46,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:47,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:51,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:58:52,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:58:55,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:58:57,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:01,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:02,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:06,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:07,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:08,634 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:59:10,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:59:13,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:15,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 04:59:15,883 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 04:59:15,883 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 04:59:15,884 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 04:59:15,935 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 04:59:16,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:16,936 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 04:59:18,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:21,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:23,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:25,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:59:28,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:31,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:33,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:36,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:38,635 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 04:59:39,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:40,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:59:44,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:46,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:49,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:51,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 04:59:54,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 04:59:55,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 04:59:59,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:01,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:04,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:06,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:08,635 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:00:10,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:00:10,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:15,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:16,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:18,464 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:00:18,466 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:00:18,466 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:00:18,467 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:00:18,971 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:00:20,972 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:00:21,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:21,468 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:25,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:00:26,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:31,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:31,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:36,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:36,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:38,636 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:00:40,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:00:41,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:46,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:46,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:51,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:00:51,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:00:55,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:00:56,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:01,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:01,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:06,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:06,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:08,640 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:01:10,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:01:11,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:16,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:16,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:21,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:01:21,246 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:01:21,247 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:01:21,248 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:01:21,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:22,004 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:01:22,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:23,005 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:01:25,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:01:27,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:31,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:32,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:36,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:37,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:38,640 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:01:40,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:01:42,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:46,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:47,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:51,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:01:52,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:01:55,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:01:57,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:01,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:02,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:06,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:07,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:08,642 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:02:10,613 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:02:13,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:16,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:18,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:21,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:23,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:02:23,255 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:02:23,256 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:02:23,274 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:02:24,037 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:02:24,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:25,038 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:02:25,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:02:29,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:31,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:34,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:36,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:38,642 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:02:39,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:40,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:02:44,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:46,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:49,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:51,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:02:54,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:02:55,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:02:59,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:01,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:04,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:06,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:08,643 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:03:10,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:03:10,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:15,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:16,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:20,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:21,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:25,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:03:25,549 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:03:25,550 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:03:25,552 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:03:25,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:03:26,071 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:03:26,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:27,071 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:03:31,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:31,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:36,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:36,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:38,645 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:03:40,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:03:41,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:46,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:46,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:51,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:03:51,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:03:55,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:03:56,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:01,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:01,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:06,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:06,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:08,644 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:04:10,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:04:12,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:16,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:17,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:21,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:22,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:25,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:04:27,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:28,241 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:04:28,243 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:04:28,243 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:04:28,245 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:04:29,107 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:04:29,107 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:04:31,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:33,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:36,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:38,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:38,645 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:04:40,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:04:43,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:46,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:48,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:51,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:04:53,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:04:55,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:04:58,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:01,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:03,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:06,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:08,646 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:05:09,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:10,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:05:14,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:16,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:19,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:21,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:24,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:25,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:05:29,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:30,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:05:30,437 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:05:30,437 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:05:30,438 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:05:30,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:31,149 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:05:31,149 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:05:35,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:36,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:38,647 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:05:40,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:05:41,155 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:05:41,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:45,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:46,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:49,161 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:05:50,161 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:05:51,496 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:05:51,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:05:55,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:05:56,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:01,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:02,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:03,169 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:06:06,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:07,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:08,648 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:06:10,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:06:12,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:15,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:17,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:20,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:22,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:25,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:06:27,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:30,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:32,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:35,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:37,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:38,649 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:06:40,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:06:43,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:45,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:48,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:50,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:06:53,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:06:55,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:06:58,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:00,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:03,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:05,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:07:05,530 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:07:05,530 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:07:05,531 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:07:05,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:06,211 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:07:07,211 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:07:08,650 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:07:09,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:10,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:07:14,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:15,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:19,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:20,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:24,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:25,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:07:29,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:30,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:34,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:35,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:38,651 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:07:40,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:40,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:07:45,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:45,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:50,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:50,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:07:55,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:07:55,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:08:00,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:00,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:05,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:05,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:08,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:08:08,376 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:08:08,377 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:08:08,378 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:08:08,652 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:08:09,248 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:08:09,248 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:08:10,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:08:11,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:15,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:16,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:20,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:21,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:25,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:08:26,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:30,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:31,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:35,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:36,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:38,653 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:08:40,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:08:42,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:45,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:47,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:50,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:08:52,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:08:55,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:08:57,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:00,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:02,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:05,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:07,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:08,654 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:09:10,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:09:10,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:09:10,737 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:09:10,737 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:09:10,738 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:09:11,285 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:09:11,286 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:09:12,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:15,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:17,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:20,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:22,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:25,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:09:27,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:30,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:32,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:35,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:37,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:38,655 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:09:40,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:09:43,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:45,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:48,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:50,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:09:53,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:09:55,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:09:58,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:00,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:03,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:05,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:08,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:08,659 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:10:10,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:10:13,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:10:13,543 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:10:13,543 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:10:13,544 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:10:14,362 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:10:14,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:15,363 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:10:15,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:19,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:20,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:24,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:25,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:10:29,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:30,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:34,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:35,843 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:38,660 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:10:39,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:40,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:10:44,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:45,844 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:49,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:50,845 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:10:54,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:10:55,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:10:59,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:00,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:04,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:05,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:08,663 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:11:10,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:10,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:11:15,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:15,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:16,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:11:16,380 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:11:16,380 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:11:16,382 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:11:16,397 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:11:17,398 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:11:20,850 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:21,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:25,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:11:26,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:30,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:31,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:35,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:36,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:38,662 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:11:40,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:11:41,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:45,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:46,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:50,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:11:51,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:11:55,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:11:56,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:00,855 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:01,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:05,856 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:06,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:08,663 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:12:10,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:12:12,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:15,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:17,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:18,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:12:18,946 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:12:18,946 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:12:18,948 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:12:19,431 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:12:19,432 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:12:20,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:22,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:25,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:12:27,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:30,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:32,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:35,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:37,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:38,663 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:12:40,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:12:43,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:45,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:48,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:50,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:12:53,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:12:55,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:12:58,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:00,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:03,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:05,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:08,664 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:13:09,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:10,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:13:14,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:15,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:19,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:21,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:21,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:13:21,526 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:13:21,527 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:13:21,528 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:13:22,467 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:13:23,467 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:13:25,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:25,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:13:30,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:31,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:35,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:36,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:38,665 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:13:40,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:40,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:13:45,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:46,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:50,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:51,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:13:55,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:13:55,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:14:00,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:01,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:05,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:06,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:08,666 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:14:10,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:14:11,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:16,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:16,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:21,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:21,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:23,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:14:23,909 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:14:23,909 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:14:23,910 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:14:24,523 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:14:25,524 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:14:25,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:14:26,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:31,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:31,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:36,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:36,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:38,667 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:14:40,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:14:42,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:46,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:47,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:51,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:14:52,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:14:55,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:14:57,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:01,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:02,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:06,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:07,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:08,668 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:15:10,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:15:13,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:16,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:18,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:21,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:23,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:25,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:15:26,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:15:26,884 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:15:26,884 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:15:26,885 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:15:27,580 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:15:27,580 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:15:28,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:31,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:33,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:36,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:38,669 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:15:39,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:40,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:15:44,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:46,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:49,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:51,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:15:54,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:15:55,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:15:59,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:01,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:04,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:06,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:08,670 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:16:10,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:10,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:16:15,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:16,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:20,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:21,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:25,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:25,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:16:29,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:16:29,514 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:16:29,514 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:16:29,515 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:16:29,624 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:16:30,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:16:31,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:31,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:31,626 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:16:36,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:36,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:38,670 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:16:40,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:16:41,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:46,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:46,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:51,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:16:51,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:16:55,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:16:56,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:01,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:01,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:06,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:06,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:08,672 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:17:10,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:17:12,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:16,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:17,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:21,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:22,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:25,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:17:27,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:31,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:32,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:32,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:17:32,939 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:17:32,939 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:17:32,941 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:17:33,682 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:17:33,682 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:17:36,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:37,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:38,672 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:17:40,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:17:43,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:46,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:48,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:51,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:17:53,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:17:55,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:17:58,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:01,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:03,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:06,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:08,673 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:18:09,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:10,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:18:14,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:16,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:19,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:21,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:24,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:25,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:18:29,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:31,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:34,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:35,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:18:35,230 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:18:35,230 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:18:35,231 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:18:35,718 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:18:35,718 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:18:36,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:38,674 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:18:40,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:40,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:18:45,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:46,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:50,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:51,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:18:55,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:18:55,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:19:00,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:01,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:05,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:06,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:08,675 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:19:10,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:19:11,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:16,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:16,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:21,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:21,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:25,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:19:26,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:31,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:31,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:36,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:36,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:37,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:19:37,979 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:19:37,979 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:19:37,980 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:19:38,676 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:19:38,755 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:19:40,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:19:40,756 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:19:42,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:46,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:47,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:51,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:19:52,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:19:55,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:19:57,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:01,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:02,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:06,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:07,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:08,677 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:20:10,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:20:13,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:16,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:18,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:21,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:23,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:25,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:20:28,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:31,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:33,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:36,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:38,678 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:20:39,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:40,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:20:40,432 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:20:40,433 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:20:40,434 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:20:40,788 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:20:40,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:20:42,789 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:20:45,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:46,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:50,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:51,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:20:55,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:20:55,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:21:00,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:01,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:05,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:06,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:08,679 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:21:10,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:10,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:21:15,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:16,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:20,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:21,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:25,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:25,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:21:30,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:31,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:35,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:36,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:38,680 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:21:40,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:21:41,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:42,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:21:42,791 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:21:42,791 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:21:42,792 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:21:42,821 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:21:44,822 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:21:46,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:46,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:51,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:21:51,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:21:55,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:21:56,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:01,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:01,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:06,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:06,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:08,681 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:22:10,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:22:12,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:16,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:17,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:21,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:22,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:25,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:22:27,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:31,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:32,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:36,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:37,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:38,682 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:22:40,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:22:43,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:45,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:22:45,457 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:22:45,457 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:22:45,458 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:22:45,855 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:22:46,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:46,856 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:22:49,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:51,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:22:54,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:22:55,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:22:59,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:01,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:04,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:06,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:08,683 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:23:09,683 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:10,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:23:14,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:16,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:19,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:21,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:24,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:25,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:23:29,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:31,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:34,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:36,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:38,683 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:23:40,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:40,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:23:45,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:46,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:47,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:23:47,745 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:23:47,745 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:23:47,747 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:23:47,888 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:23:48,889 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:23:50,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:51,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:23:55,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:23:55,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:24:00,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:01,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:05,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:06,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:08,684 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:24:10,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:24:11,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:16,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:16,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:21,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:21,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:25,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:24:26,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:31,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:31,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:36,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:36,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:38,685 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:24:40,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:24:42,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:46,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:47,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:50,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:24:50,272 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:24:50,272 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:24:50,273 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:24:50,921 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:24:51,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:24:52,923 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:24:53,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:24:55,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:24:58,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:01,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:03,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:06,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:08,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:08,686 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:25:10,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:25:13,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:16,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:18,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:21,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:23,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:25,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:25:28,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:31,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:33,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:36,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:38,687 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:25:39,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:40,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:25:44,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:46,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:49,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:51,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:25:52,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:25:52,985 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:25:52,986 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:25:52,987 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:25:53,958 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:25:54,958 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:25:54,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:25:55,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:25:59,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:01,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:04,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:06,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:08,691 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:26:10,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:10,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:26:15,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:16,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:20,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:21,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:25,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:25,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:26:30,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:31,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:35,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:36,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:38,689 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:26:40,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:26:41,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:46,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:46,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:51,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:26:51,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:26:55,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:26:55,214 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:26:55,214 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:26:55,217 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:26:55,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:26:55,995 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:26:56,996 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:26:57,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:01,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:02,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:06,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:07,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:08,690 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:27:10,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:27:12,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:16,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:17,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:21,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:22,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:25,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:27:27,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:31,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:32,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:36,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:37,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:38,692 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:27:40,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:27:43,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:46,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:48,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:51,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:27:53,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:27:55,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:27:58,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:27:58,047 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:27:58,047 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:27:58,048 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:27:59,029 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:27:59,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:01,031 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:28:01,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:04,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:06,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:08,692 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:28:09,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:10,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:28:14,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:16,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:19,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:21,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:24,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:25,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:28:29,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:31,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:34,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:36,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:38,693 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:28:40,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:40,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:28:45,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:46,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:50,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:51,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:28:55,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:28:55,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:29:00,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:29:00,573 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:29:00,573 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:29:00,575 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:29:01,063 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:29:01,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:01,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:03,065 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:29:06,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:06,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:08,693 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:29:10,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:29:11,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:16,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:16,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:21,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:21,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:25,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:29:26,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:31,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:31,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:36,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:36,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:38,694 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:29:40,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:29:42,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:46,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:47,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:51,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:29:52,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:29:55,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:29:57,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:00,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:02,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:03,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:30:03,197 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:30:03,197 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:30:03,199 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:30:04,098 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:30:05,099 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:30:05,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:08,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:08,695 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:30:10,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:30:13,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:15,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:18,696 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:20,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:23,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:25,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:30:28,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:30,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:33,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:35,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:38,696 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:30:39,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:40,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:30:44,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:45,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:49,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:50,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:30:54,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:30:55,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:30:59,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:00,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:04,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:05,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:31:05,600 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:31:05,600 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:31:05,602 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:31:05,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:06,134 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:31:07,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:31:08,697 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:31:10,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:10,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:31:15,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:15,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:20,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:20,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:25,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:25,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:31:30,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:30,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:35,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:35,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:38,698 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:31:40,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:31:41,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:45,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:46,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:50,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:31:51,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:31:55,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:31:56,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:00,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:01,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:05,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:06,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:08,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:32:08,192 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:32:08,193 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:32:08,194 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:32:08,698 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:32:09,170 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:32:09,170 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:32:10,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:32:12,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:15,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:17,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:20,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:22,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:25,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:32:27,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:30,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:32,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:35,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:37,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:38,699 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:32:40,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:32:43,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:45,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:48,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:50,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:32:53,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:32:55,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:32:58,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:00,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:03,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:05,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:08,700 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:33:09,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:10,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:33:11,048 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:33:11,050 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:33:11,050 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:33:11,051 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:33:11,204 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:33:13,206 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:33:15,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:15,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:20,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:20,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:25,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:25,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:33:30,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:30,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:35,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:35,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:38,704 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:33:40,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:40,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:33:45,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:45,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:50,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:50,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:33:55,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:33:55,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:34:00,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:00,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:05,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:05,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:08,702 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:34:10,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:34:11,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:13,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:34:13,267 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:34:13,267 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:34:13,269 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:34:14,242 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:34:15,243 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:34:15,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:17,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:20,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:22,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:25,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:34:27,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:30,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:32,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:35,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:37,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:38,703 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:34:40,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:34:42,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:45,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:47,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:50,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:34:52,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:34:55,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:34:57,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:00,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:02,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:05,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:07,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:08,705 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:35:10,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:35:13,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:15,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:35:15,524 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:35:15,525 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:35:15,526 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:35:15,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:16,274 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:35:17,275 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:35:19,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:20,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:24,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:25,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:35:29,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:30,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:34,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:35,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:38,705 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:35:39,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:40,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:35:44,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:45,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:49,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:50,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:35:54,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:35:55,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:35:59,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:00,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:04,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:05,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:08,706 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:36:10,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:10,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:36:15,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:15,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:18,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:36:18,158 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:36:18,159 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:36:18,160 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:36:18,308 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:36:19,308 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:36:20,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:21,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:25,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:36:26,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:30,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:31,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:35,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:36,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:38,707 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:36:40,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:36:41,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:45,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:46,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:50,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:36:51,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:36:55,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:36:56,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:00,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:01,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:05,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:06,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:08,707 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:37:10,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:37:12,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:15,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:17,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:20,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:37:20,299 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:37:20,299 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:37:20,300 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:37:20,341 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:37:20,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:37:20,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:21,342 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:37:22,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:23,343 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:37:25,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:37:27,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:30,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:32,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:35,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:37,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:38,708 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:37:40,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:37:43,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:45,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:48,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:50,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:37:53,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:37:55,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:37:58,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:00,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:03,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:05,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:08,709 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:38:09,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:10,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:38:14,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:15,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:19,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:20,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:23,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:38:23,656 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:38:23,657 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:38:23,658 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:38:24,377 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:38:25,377 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:38:25,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:25,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:38:30,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:30,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:35,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:35,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:38,710 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:38:40,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:40,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:38:45,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:45,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:50,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:50,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:38:55,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:38:55,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:39:00,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:00,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:05,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:05,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:08,711 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:39:10,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:39:11,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:15,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:16,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:21,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:21,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:25,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:39:26,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:39:26,115 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:39:26,116 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:39:26,117 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:39:26,435 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:39:27,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:27,436 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:39:31,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:32,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:36,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:37,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:38,711 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:39:40,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:39:42,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:46,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:47,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:51,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:39:52,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:39:55,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:39:57,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:01,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:02,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:06,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:07,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:08,712 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:40:10,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:40:13,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:16,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:18,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:21,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:23,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:25,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:40:28,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:40:28,301 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:40:28,301 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:40:28,303 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:40:28,469 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:40:29,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:29,470 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:40:31,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:34,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:36,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:38,713 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:40:39,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:40,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:40:44,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:46,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:49,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:51,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:40:54,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:40:55,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:40:59,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:01,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:04,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:06,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:08,714 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:41:10,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:10,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:41:15,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:16,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:20,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:21,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:25,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:25,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:41:30,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:31,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:31,142 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:41:31,147 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:41:31,147 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:41:31,148 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:41:31,509 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:41:33,510 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:41:36,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:36,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:38,715 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:41:40,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:41:41,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:46,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:46,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:51,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:41:51,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:41:55,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:41:56,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:01,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:01,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:06,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:06,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:08,716 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:42:10,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:42:12,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:16,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:17,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:21,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:22,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:25,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:42:27,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:31,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:32,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:33,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:42:33,832 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:42:33,832 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:42:33,834 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:42:34,552 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:42:35,553 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:42:36,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:37,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:38,717 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:42:40,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:42:43,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:46,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:48,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:51,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:42:53,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:42:55,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:42:58,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:01,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:03,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:06,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:08,718 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:43:09,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:10,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:43:14,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:16,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:19,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:21,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:24,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:25,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:43:29,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:31,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:34,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:35,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:43:35,809 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:43:35,810 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:43:35,811 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:43:36,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:36,587 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:43:37,587 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:43:38,719 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:43:40,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:40,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:43:45,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:46,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:50,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:51,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:43:55,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:43:55,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:44:00,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:01,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:05,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:06,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:08,720 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:44:10,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:44:11,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:16,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:16,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:21,240 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:21,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:25,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:44:26,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:31,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:31,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:36,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:36,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:38,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:44:38,535 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:44:38,535 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:44:38,537 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:44:38,619 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:44:38,720 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:44:39,620 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:44:40,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:44:42,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:46,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:47,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:51,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:44:52,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:44:55,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:44:57,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:01,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:02,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:06,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:07,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:08,721 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:45:10,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:45:13,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:16,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:18,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:21,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:23,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:25,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:45:28,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:31,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:33,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:36,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:38,722 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:45:39,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:40,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:45:41,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:45:41,161 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:45:41,161 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:45:41,162 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:45:41,654 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:45:43,655 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:45:45,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:46,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:50,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:51,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:45:55,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:45:55,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:46:00,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:01,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:05,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:06,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:08,723 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:46:10,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:10,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:46:15,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:16,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:20,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:21,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:25,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:25,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:46:30,726 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:31,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:35,726 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:36,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:38,724 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:46:40,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:46:41,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:43,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:46:43,887 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:46:43,887 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:46:43,888 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:46:44,687 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:46:45,687 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:46:46,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:46,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:51,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:46:51,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:46:55,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:46:56,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:01,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:01,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:06,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:06,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:08,725 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:47:10,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:47:12,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:16,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:17,726 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:21,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:22,726 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:25,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:47:27,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:31,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:32,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:36,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:37,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:38,725 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:47:40,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:47:43,726 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:45,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:47:45,919 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:47:45,919 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:47:45,920 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:47:46,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:46,723 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:47:47,724 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:47:48,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:51,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:47:53,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:47:55,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:47:58,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:01,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:03,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:06,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:08,726 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:48:09,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:10,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:48:14,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:16,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:19,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:21,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:24,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:25,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:48:29,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:31,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:34,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:36,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:38,727 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:48:40,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:40,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:48:45,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:46,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:48,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:48:48,295 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:48:48,296 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:48:48,297 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:48:48,758 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:48:49,759 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:48:51,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:48:51,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:48:55,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:48:56,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:01,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:01,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:06,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:06,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:08,728 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:49:10,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:49:11,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:16,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:16,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:21,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:21,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:25,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:49:26,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:31,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:31,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:36,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:36,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:38,729 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:49:40,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:49:42,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:46,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:47,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:50,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:49:50,556 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:49:50,557 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:49:50,558 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:49:50,823 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:49:51,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:49:51,824 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:49:53,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:49:55,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:49:58,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:01,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:03,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:06,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:08,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:08,730 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:50:10,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:50:13,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:16,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:18,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:21,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:23,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:25,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:50:28,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:31,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:33,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:36,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:38,731 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:50:39,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:40,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:50:44,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:46,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:49,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:51,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:50:52,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:50:52,859 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:50:52,860 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:50:52,862 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:50:53,862 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:50:53,862 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:50:54,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:50:55,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:50:59,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:01,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:04,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:06,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:08,732 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:51:10,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:10,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:51:15,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:16,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:20,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:21,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:25,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:25,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:51:30,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:31,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:35,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:36,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:38,733 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:51:40,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:51:41,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:46,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:46,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:51,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:51,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:51:55,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:51:55,660 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:51:55,661 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:51:55,662 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:51:55,897 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:51:55,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:51:57,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:51:57,898 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:52:01,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:02,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:06,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:07,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:08,734 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:52:10,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:52:12,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:16,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:17,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:21,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:22,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:25,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:52:27,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:31,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:32,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:36,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:37,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:38,735 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:52:40,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:52:43,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:46,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:48,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:51,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:52:53,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:55,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:52:57,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:52:57,864 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:52:57,865 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:52:57,866 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:52:57,970 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:52:58,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:52:59,972 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:53:00,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:03,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:05,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:08,736 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:53:09,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:10,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:53:14,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:15,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:19,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:20,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:24,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:25,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:53:29,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:30,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:34,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:35,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:38,738 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:53:40,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:40,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:53:45,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:45,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:50,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:50,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:53:55,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:53:55,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:54:00,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:54:00,425 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:54:00,425 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:54:00,426 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:54:00,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:01,004 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:54:01,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:02,005 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:54:05,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:06,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:08,737 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:54:10,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:54:11,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:15,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:16,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:20,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:21,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:25,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:54:26,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:30,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:31,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:35,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:36,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:38,739 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:54:40,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:54:42,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:45,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:47,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:50,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:54:52,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:54:55,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:54:57,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:00,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:02,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:02,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:55:02,893 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:55:02,893 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:55:02,894 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:55:03,044 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:55:04,045 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:55:05,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:07,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:08,739 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:55:10,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:55:13,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:15,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:18,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:20,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:23,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:25,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:55:28,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:30,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:33,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:35,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:38,740 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:55:39,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:40,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:55:44,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:46,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:49,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:51,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:55:54,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:55:55,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:55:59,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:01,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:04,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:05,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:56:05,345 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:56:05,346 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:56:05,348 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:56:06,046 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:06,079 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:56:06,079 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:56:08,741 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:56:10,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:10,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:56:15,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:16,050 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:20,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:21,051 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:25,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:25,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:56:30,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:31,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:35,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:36,054 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:38,744 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:56:40,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:56:41,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:46,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:46,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:51,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:56:51,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:56:55,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:56:56,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:01,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:01,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:06,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:06,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:07,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:57:07,876 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:57:07,876 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:57:07,878 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:57:08,142 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:57:08,142 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:57:08,743 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:57:10,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:57:12,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:16,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:17,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:21,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:22,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:25,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:57:27,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:31,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:32,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:36,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:37,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:38,745 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:57:40,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:57:43,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:46,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:48,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:51,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:57:53,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:57:55,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:57:58,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:01,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:03,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:06,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:08,745 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:58:09,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:10,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:58:10,068 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:58:10,068 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:58:10,070 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:58:10,176 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:58:11,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:11,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:58:12,177 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:58:15,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:16,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:21,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:21,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:22,183 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:58:26,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:26,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:58:28,187 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:58:30,188 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:58:31,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:32,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:36,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:37,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:38,746 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:58:41,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:58:42,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:42,196 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:58:43,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:58:44,197 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:58:46,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:48,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:51,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:58:53,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:58:56,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:58:58,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:01,840 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:03,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:06,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:08,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:08,746 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:59:11,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:59:13,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:16,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:18,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:21,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:23,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:26,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:59:28,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:31,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:33,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:36,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:38,748 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 05:59:39,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:41,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 05:59:44,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:46,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:47,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 05:59:47,127 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 05:59:47,127 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 05:59:47,129 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 05:59:47,243 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 05:59:49,244 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 05:59:50,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:51,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 05:59:55,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 05:59:56,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:00:00,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:01,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:05,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:06,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:08,749 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:00:10,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:11,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:00:15,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:16,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:20,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:21,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:25,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:26,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:00:30,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:32,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:35,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:37,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:38,750 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:00:41,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:00:41,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:46,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:47,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:50,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:00:50,117 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:00:50,117 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:00:50,119 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:00:50,349 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:00:50,350 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:00:52,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:00:52,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:00:56,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:00:57,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:02,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:02,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:07,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:07,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:08,750 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:01:11,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:01:12,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:17,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:17,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:22,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:22,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:26,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:01:27,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:32,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:32,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:37,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:37,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:38,751 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:01:41,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:01:43,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:47,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:48,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:52,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:01:52,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:01:52,147 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:01:52,147 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:01:52,149 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:01:52,384 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:01:52,384 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:01:54,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:01:56,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:01:59,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:02,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:04,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:07,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:08,752 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:02:09,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:11,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:02:14,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:17,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:19,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:22,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:24,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:26,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:02:29,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:32,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:34,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:37,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:38,753 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:02:40,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:41,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:02:45,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:47,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:50,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:52,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:02:54,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:02:54,832 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:02:54,832 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:02:54,834 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:02:55,423 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:02:55,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:02:56,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:02:56,423 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:03:00,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:02,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:05,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:07,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:08,754 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:03:11,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:03:11,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:16,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:17,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:21,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:22,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:26,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:03:26,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:31,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:32,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:36,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:37,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:38,757 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:03:41,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:03:41,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:46,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:47,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:51,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:52,226 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:03:56,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:03:56,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:03:57,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:03:57,313 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:03:57,313 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:03:57,314 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:03:57,456 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:03:59,457 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:04:02,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:02,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:07,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:07,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:08,756 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:04:11,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:04:12,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:17,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:17,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:22,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:22,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:26,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:04:27,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:32,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:32,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:37,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:37,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:38,758 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:04:41,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:04:43,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:47,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:48,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:52,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:04:53,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:56,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:04:58,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:04:59,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:04:59,736 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:04:59,736 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:04:59,738 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:05:00,490 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:05:01,491 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:05:02,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:03,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:07,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:08,758 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:05:09,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:11,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:05:14,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:17,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:19,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:22,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:24,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:26,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:05:29,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:32,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:34,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:37,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:38,758 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:05:40,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:41,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:05:45,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:46,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:50,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:51,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:05:55,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:05:56,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:06:00,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:01,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:02,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:06:02,336 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:06:02,336 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:06:02,338 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:06:02,528 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:06:03,528 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:06:06,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:06,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:08,759 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:06:11,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:06:11,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:16,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:16,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:21,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:21,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:26,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:06:26,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:31,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:31,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:36,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:36,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:38,760 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:06:41,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:06:42,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:46,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:47,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:51,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:06:52,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:06:56,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:06:57,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:01,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:02,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:05,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:07:05,087 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:07:05,087 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:07:05,089 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:07:05,564 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:07:06,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:07,565 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:07:08,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:08,761 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:07:11,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:07:13,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:16,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:18,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:21,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:23,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:26,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:07:28,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:31,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:33,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:36,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:38,762 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:07:39,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:41,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:07:44,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:46,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:49,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:51,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:07:54,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:07:56,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:07:59,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:01,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:04,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:06,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:07,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:08:07,751 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:08:07,751 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:08:07,753 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:08:08,600 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:08:08,762 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:08:09,600 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:08:10,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:11,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:08:15,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:16,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:20,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:21,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:25,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:26,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:08:30,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:31,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:35,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:36,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:38,764 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:08:41,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:08:41,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:46,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:46,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:51,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:08:51,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:08:56,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:08:56,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:01,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:01,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:06,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:06,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:08,764 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:09:10,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:09:10,613 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:09:10,613 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:09:10,615 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:09:10,637 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:09:11,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:09:11,638 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:09:12,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:16,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:17,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:21,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:22,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:26,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:09:27,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:31,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:32,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:36,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:37,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:38,765 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:09:41,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:09:42,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:46,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:47,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:51,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:09:52,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:09:56,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:09:57,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:01,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:02,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:06,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:07,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:08,766 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:10:11,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:10:13,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:10:13,218 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:10:13,219 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:10:13,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:13,220 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:10:13,674 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:10:15,675 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:10:16,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:18,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:21,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:23,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:26,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:10:28,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:31,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:33,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:36,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:38,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:38,767 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:10:41,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:10:43,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:46,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:48,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:51,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:10:53,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:10:56,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:10:58,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:01,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:03,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:06,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:08,768 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:11:09,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:11,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:11:14,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:15,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:11:15,867 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:11:15,867 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:11:15,869 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:11:16,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:16,762 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:11:17,763 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:11:19,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:21,692 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:24,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:26,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:11:29,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:31,693 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:34,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:36,694 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:38,769 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:11:40,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:41,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:11:45,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:46,695 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:50,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:51,697 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:11:55,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:11:56,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:12:00,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:01,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:05,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:06,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:08,770 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:12:11,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:12:11,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:16,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:16,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:18,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:12:18,393 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:12:18,394 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:12:18,395 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:12:18,798 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:12:19,799 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:12:21,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:22,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:26,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:12:27,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:31,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:32,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:36,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:37,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:38,771 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:12:41,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:12:42,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:46,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:47,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:51,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:12:52,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:12:56,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:12:57,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:01,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:02,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:06,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:07,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:08,772 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:13:11,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:13:13,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:16,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:18,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:21,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:13:21,303 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:13:21,303 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:13:21,304 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:13:21,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:21,835 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:13:23,836 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:13:24,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:26,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:13:29,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:31,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:34,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:36,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:38,773 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:13:39,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:41,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:13:44,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:46,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:49,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:51,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:13:54,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:13:56,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:13:59,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:01,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:04,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:06,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:08,774 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:14:10,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:11,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:14:15,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:16,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:20,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:21,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:23,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:14:23,173 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:14:23,174 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:14:23,175 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:14:23,869 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:14:25,870 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:14:26,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:26,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:14:31,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:31,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:36,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:36,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:38,775 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:14:41,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:14:41,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:46,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:46,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:51,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:14:51,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:14:56,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:14:56,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:01,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:01,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:06,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:06,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:08,775 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:15:11,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:15:12,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:16,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:17,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:21,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:22,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:25,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:15:25,720 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:15:25,720 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:15:25,721 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:15:25,903 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:15:26,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:15:27,910 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:15:28,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:31,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:33,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:36,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:38,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:38,776 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:15:41,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:15:43,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:46,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:48,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:51,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:15:53,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:15:56,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:15:58,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:01,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:03,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:06,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:08,777 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:16:09,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:11,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:16:14,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:16,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:19,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:21,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:24,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:26,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:16:28,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:16:28,291 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:16:28,291 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:16:28,293 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:16:28,943 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:16:29,944 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:16:30,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:31,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:35,331 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:36,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:38,778 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:16:40,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:41,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:16:45,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:46,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:50,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:51,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:16:55,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:16:56,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:17:00,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:01,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:05,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:06,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:08,779 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:17:11,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:17:11,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:16,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:16,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:21,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:21,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:26,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:17:26,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:31,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:17:31,092 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:17:31,092 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:17:31,093 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:17:31,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:31,980 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:17:31,980 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:17:32,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:36,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:37,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:38,780 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:17:41,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:17:42,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:46,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:47,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:51,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:17:52,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:17:56,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:17:57,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:01,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:02,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:06,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:07,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:08,781 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:18:11,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:18:13,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:16,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:18,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:21,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:23,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:26,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:18:28,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:31,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:33,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:18:33,379 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:18:33,380 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:18:33,381 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:18:34,014 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:18:34,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:36,015 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:18:36,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:38,784 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:18:39,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:41,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:18:44,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:46,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:49,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:51,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:18:54,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:18:56,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:18:59,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:01,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:04,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:07,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:08,783 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:19:10,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:11,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:19:15,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:17,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:20,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:22,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:25,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:26,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:19:30,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:32,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:35,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:36,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:19:36,132 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:19:36,132 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:19:36,134 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:19:36,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:19:37,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:37,048 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:19:38,049 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:19:38,783 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:19:41,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:19:41,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:46,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:47,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:51,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:19:52,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:19:56,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:19:56,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:01,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:02,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:06,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:07,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:08,785 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:20:11,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:20:12,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:17,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:17,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:22,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:22,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:26,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:20:27,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:32,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:32,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:37,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:37,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:38,787 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:20:39,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:20:39,484 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:20:39,485 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:20:39,486 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:20:40,091 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:20:41,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:20:42,092 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:20:43,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:47,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:48,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:52,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:20:53,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:20:56,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:20:58,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:02,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:03,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:07,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:08,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:08,787 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:21:11,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:21:13,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:17,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:18,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:22,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:23,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:26,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:21:28,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:32,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:33,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:37,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:38,788 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:21:39,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:41,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:21:42,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:21:42,261 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:21:42,261 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:21:42,262 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:21:43,125 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:21:44,125 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:21:45,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:47,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:50,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:52,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:21:55,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:21:56,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:22:00,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:02,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:05,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:07,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:08,789 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:22:10,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:11,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:22:15,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:17,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:20,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:22,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:25,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:26,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:22:30,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:32,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:35,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:37,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:38,790 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:22:41,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:22:41,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:44,466 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:22:44,468 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:22:44,468 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:22:44,469 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:22:45,195 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:22:46,195 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:22:47,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:47,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:52,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:22:52,511 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:22:56,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:22:57,512 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:02,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:02,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:07,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:07,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:08,791 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:23:11,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:23:12,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:17,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:17,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:22,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:22,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:26,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:23:27,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:32,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:32,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:37,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:37,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:38,791 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:23:41,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:23:43,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:47,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:23:47,133 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:23:47,133 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:23:47,134 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:23:47,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:47,227 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:23:48,227 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:23:49,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:52,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:23:54,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:23:56,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:23:59,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:02,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:04,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:07,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:08,792 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:24:09,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:11,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:24:14,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:17,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:19,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:22,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:24,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:26,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:24:29,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:32,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:34,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:37,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:38,793 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:24:40,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:41,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:24:45,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:47,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:49,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:24:49,551 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:24:49,552 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:24:49,553 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:24:50,260 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:24:50,261 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:24:51,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:24:52,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:24:56,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:24:56,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:01,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:02,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:06,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:07,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:08,794 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:25:11,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:25:11,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:16,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:17,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:21,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:22,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:26,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:25:26,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:31,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:32,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:36,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:37,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:38,795 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:25:41,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:25:42,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:47,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:47,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:52,044 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:25:52,049 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:25:52,050 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:25:52,051 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:25:52,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:25:52,294 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:25:53,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:25:54,295 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:25:56,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:25:58,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:02,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:03,053 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:07,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:08,053 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:08,796 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:26:11,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:26:13,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:17,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:18,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:22,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:23,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:26,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:26:28,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:32,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:33,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:37,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:38,799 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:26:38,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:41,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:26:43,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:47,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:48,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:52,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:26:53,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:26:54,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:26:54,055 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:26:54,055 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:26:54,056 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:26:54,326 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:26:56,327 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:26:56,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:26:59,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:01,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:04,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:06,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:08,797 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:27:09,798 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:11,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:27:14,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:16,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:19,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:21,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:24,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:26,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:27:29,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:31,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:34,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:36,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:38,799 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:27:40,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:41,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:27:45,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:46,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:50,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:51,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:27:55,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:27:56,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:27:56,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:27:56,598 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:27:56,599 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:27:56,600 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:27:57,360 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:27:58,361 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:28:01,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:01,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:06,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:06,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:08,799 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:28:11,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:28:11,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:16,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:16,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:21,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:21,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:26,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:28:26,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:31,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:31,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:36,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:36,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:38,800 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:28:41,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:28:42,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:46,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:47,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:51,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:28:52,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:56,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:28:57,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:28:59,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:28:59,013 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:28:59,013 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:28:59,014 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:28:59,393 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:29:00,394 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:29:01,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:03,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:06,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:08,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:08,801 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:29:11,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:29:13,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:16,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:18,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:21,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:23,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:26,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:29:28,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:31,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:33,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:36,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:38,802 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:29:39,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:41,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:29:44,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:46,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:49,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:51,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:29:54,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:29:56,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:29:59,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:01,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:30:01,372 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:30:01,372 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:30:01,373 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:30:01,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:01,426 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:30:02,427 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:30:05,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:06,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:08,803 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:30:10,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:11,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:30:15,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:16,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:20,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:21,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:25,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:26,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:30:30,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:31,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:35,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:36,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:38,804 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:30:41,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:30:41,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:46,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:46,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:51,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:30:51,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:30:56,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:30:56,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:01,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:01,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:04,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:31:04,405 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:31:04,406 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:31:04,407 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:31:04,460 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:31:06,461 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:31:06,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:07,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:08,805 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:31:11,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:31:12,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:16,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:17,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:21,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:22,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:26,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:31:27,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:31,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:32,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:36,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:37,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:38,806 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:31:41,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:31:43,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:46,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:48,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:51,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:31:53,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:31:56,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:31:58,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:01,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:03,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:06,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:07,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:32:07,226 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:32:07,227 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:32:07,228 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:32:07,493 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:32:08,494 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:32:08,807 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:32:09,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:11,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:32:14,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:16,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:19,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:21,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:24,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:26,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:32:29,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:31,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:34,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:36,587 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:38,808 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:32:40,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:41,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:32:45,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:46,588 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:50,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:51,589 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:32:55,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:32:56,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:33:00,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:01,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:05,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:06,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:08,808 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:33:10,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:33:10,099 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:33:10,099 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:33:10,100 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:33:10,527 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:33:10,527 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:33:11,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:11,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:33:16,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:16,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:21,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:21,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:26,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:26,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:33:31,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:31,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:36,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:36,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:38,809 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:33:41,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:33:41,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:46,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:46,810 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:51,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:33:51,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:33:56,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:33:56,811 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:01,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:01,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:06,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:06,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:08,810 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:34:11,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:34:12,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:34:12,119 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:34:12,119 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:34:12,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:12,120 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:34:12,560 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:34:12,561 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:34:16,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:17,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:21,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:22,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:26,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:34:27,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:31,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:32,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:36,605 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:37,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:38,811 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:34:41,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:34:42,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:46,698 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:47,812 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:51,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:34:52,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:34:56,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:34:57,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:01,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:02,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:06,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:07,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:08,812 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:35:11,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:35:13,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:14,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:35:14,755 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:35:14,755 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:35:14,759 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:35:15,594 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:35:16,595 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:35:16,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:19,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:21,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:24,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:26,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:35:29,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:31,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:34,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:36,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:38,813 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:35:39,813 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:41,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:35:44,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:46,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:49,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:51,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:35:54,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:35:56,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:35:59,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:01,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:04,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:06,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:08,816 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:36:10,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:11,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:36:15,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:16,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:17,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:36:17,354 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:36:17,354 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:36:17,355 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:36:17,627 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:36:18,627 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:36:21,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:21,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:26,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:36:26,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:31,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:31,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:36,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:36,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:38,817 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:36:41,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:36:41,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:46,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:46,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:51,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:36:51,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:36:56,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:36:56,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:01,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:01,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:06,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:06,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:08,818 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:37:11,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:37:12,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:16,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:17,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:19,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:37:19,441 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:37:19,442 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:37:19,443 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:37:19,661 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:37:20,662 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:37:21,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:23,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:26,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:37:28,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:31,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:33,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:36,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:38,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:38,818 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:37:41,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:37:43,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:46,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:48,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:51,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:37:53,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:37:56,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:37:58,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:01,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:03,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:06,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:08,819 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:38:09,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:11,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:38:14,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:16,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:19,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:21,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:38:21,761 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:38:21,761 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:38:21,763 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:38:21,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:22,696 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:38:22,696 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:38:25,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:26,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:38:30,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:31,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:35,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:36,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:38,820 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:38:40,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:41,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:38:45,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:46,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:50,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:51,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:38:55,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:38:56,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:39:00,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:01,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:05,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:06,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:08,821 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:39:11,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:39:11,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:16,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:16,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:21,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:21,839 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:24,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:39:24,823 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:39:24,823 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:39:24,825 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:39:25,731 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:39:26,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:39:26,732 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:39:26,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:31,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:31,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:36,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:36,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:38,822 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:39:41,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:39:42,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:46,843 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:47,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:51,844 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:39:52,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:39:56,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:39:57,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:01,846 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:02,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:06,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:07,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:08,823 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:40:11,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:40:13,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:16,848 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:18,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:21,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:23,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:26,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:40:27,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:40:27,642 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:40:27,642 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:40:27,644 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:40:27,765 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:40:28,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:40:28,766 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:40:29,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:31,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:34,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:36,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:38,824 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:40:39,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:41,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:40:44,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:46,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:49,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:51,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:40:54,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:40:56,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:40:59,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:01,855 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:04,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:06,856 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:08,825 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:41:10,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:11,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:41:15,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:16,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:20,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:21,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:25,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:26,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:41:30,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:41:30,624 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:41:30,624 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:41:30,626 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:41:30,800 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:41:31,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:31,801 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:41:31,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:36,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:36,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:38,827 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:41:41,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:41:41,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:46,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:46,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:51,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:41:51,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:41:56,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:41:56,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:01,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:01,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:06,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:06,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:08,826 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:42:11,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:42:12,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:16,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:17,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:21,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:22,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:26,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:42:27,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:31,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:32,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:33,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:42:33,305 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:42:33,305 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:42:33,308 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:42:33,836 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:42:35,837 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:42:36,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:38,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:38,827 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:42:41,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:42:43,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:46,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:48,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:51,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:42:53,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:42:56,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:42:58,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:01,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:03,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:06,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:08,829 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:43:09,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:11,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:43:14,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:16,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:19,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:21,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:24,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:26,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:43:29,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:31,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:34,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:36,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:43:36,135 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:43:36,135 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:43:36,137 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:43:36,871 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:43:36,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:37,872 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:43:38,829 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:43:40,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:41,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:43:45,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:46,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:50,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:51,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:43:55,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:43:56,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:44:00,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:01,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:05,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:06,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:08,830 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:44:11,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:44:11,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:16,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:16,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:21,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:21,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:26,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:44:26,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:31,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:31,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:36,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:36,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:38,831 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:44:38,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:44:38,936 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:44:38,937 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:44:38,938 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:44:39,907 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:44:41,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:44:41,908 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:44:41,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:46,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:46,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:51,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:44:51,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:44:56,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:44:56,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:01,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:01,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:06,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:06,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:08,832 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:45:11,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:45:12,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:16,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:17,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:21,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:22,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:26,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:45:27,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:31,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:32,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:36,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:37,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:38,832 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:45:41,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:45:41,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:45:41,567 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:45:41,568 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:45:41,569 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:45:41,941 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:45:43,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:43,942 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:45:46,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:48,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:51,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:45:53,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:45:56,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:45:58,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:01,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:03,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:06,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:08,610 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:08,833 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:46:11,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:46:13,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:16,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:18,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:21,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:23,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:26,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:46:28,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:31,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:33,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:36,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:38,859 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:38,860 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:46:41,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:46:43,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:44,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:46:44,175 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:46:44,176 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:46:44,177 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:46:44,976 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:46:45,977 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:46:46,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:49,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:51,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:46:54,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:46:56,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:46:59,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:01,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:04,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:06,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:08,861 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:47:09,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:11,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:47:14,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:16,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:19,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:21,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:24,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:26,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:47:29,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:31,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:34,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:36,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:38,862 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:47:40,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:41,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:47:45,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:46,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:47:46,867 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:47:46,867 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:47:46,868 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:47:46,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:47,011 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:47:48,012 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:47:50,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:51,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:47:55,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:47:56,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:48:00,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:01,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:05,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:06,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:08,863 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:48:11,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:48:11,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:16,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:16,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:21,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:21,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:26,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:48:26,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:31,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:31,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:36,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:36,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:38,864 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:48:41,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:48:42,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:46,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:47,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:49,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:48:49,259 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:48:49,259 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:48:49,261 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:48:50,047 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:48:51,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:48:52,048 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:48:53,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:48:56,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:48:58,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:02,053 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:03,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:07,054 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:08,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:08,867 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:49:11,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:49:13,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:17,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:18,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:22,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:23,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:26,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:49:28,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:32,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:33,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:37,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:38,866 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:49:39,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:41,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:49:44,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:47,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:49,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:51,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:49:51,936 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:49:51,937 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:49:51,939 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:49:52,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:49:52,082 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:49:54,083 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:49:54,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:49:56,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:49:59,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:02,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:04,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:07,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:08,866 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:50:10,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:11,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:50:15,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:17,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:20,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:22,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:25,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:26,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:50:30,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:32,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:35,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:37,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:38,868 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:50:41,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:50:41,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:46,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:47,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:51,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:50:52,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:50:54,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:50:54,950 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:50:54,951 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:50:54,952 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:50:55,117 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:50:56,118 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:50:56,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:50:57,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:01,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:02,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:06,124 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:51:06,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:07,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:08,868 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:51:11,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:51:13,128 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:51:13,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:14,129 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:51:17,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:18,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:22,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:23,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:26,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:51:28,139 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:51:29,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:30,141 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:51:31,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:34,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:36,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:38,869 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:51:39,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:41,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:51:44,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:46,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:49,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:51,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:51:54,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:51:56,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:51:59,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:01,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:04,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:06,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:08,870 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:52:10,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:11,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:52:15,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:16,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:20,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:21,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:25,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:26,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:52:30,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:31,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:31,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:52:31,922 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:52:31,923 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:52:31,924 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:52:32,182 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:52:34,184 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:52:35,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:36,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:38,871 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:52:41,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:52:41,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:46,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:46,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:51,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:52:51,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:52:56,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:52:56,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:01,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:01,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:06,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:06,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:08,872 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:53:11,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:53:12,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:16,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:17,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:21,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:22,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:26,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:53:27,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:31,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:32,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:34,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:53:34,525 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:53:34,526 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:53:34,530 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:53:35,220 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:53:36,220 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:53:36,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:38,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:38,872 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:53:41,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:53:43,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:46,771 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:48,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:51,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:53:53,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:53:56,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:53:58,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:01,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:03,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:06,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:08,873 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:54:09,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:11,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:54:14,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:16,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:19,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:21,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:24,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:26,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:54:29,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:31,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:34,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:36,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:37,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:54:37,292 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:54:37,292 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:54:37,294 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:54:38,256 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:54:38,256 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:54:38,874 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:54:40,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:41,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:54:45,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:46,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:50,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:51,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:54:55,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:54:56,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:55:00,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:01,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:05,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:06,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:08,875 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:55:11,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:55:11,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:16,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:16,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:21,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:21,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:26,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:55:26,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:31,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:31,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:36,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:36,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:38,876 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:55:40,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:55:40,299 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:55:40,299 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:55:40,300 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:55:40,301 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:55:41,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:55:42,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:42,302 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:55:46,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:47,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:51,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:55:52,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:55:56,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:55:57,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:01,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:02,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:06,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:07,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:08,877 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:56:11,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:56:12,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:16,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:17,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:21,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:22,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:26,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:56:27,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:31,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:32,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:36,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:37,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:38,880 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:56:41,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:56:42,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:56:42,836 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:56:42,836 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:56:42,837 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:56:43,337 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:56:43,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:44,338 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:56:46,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:48,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:51,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:56:53,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:56:56,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:56:58,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:01,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:03,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:06,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:08,879 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:57:09,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:11,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:57:14,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:16,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:19,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:21,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:24,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:26,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:57:29,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:31,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:34,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:37,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:38,880 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:57:40,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:41,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:57:45,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:57:45,448 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:57:45,448 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:57:45,449 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:57:46,373 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:57:46,373 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:57:46,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:47,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:51,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:57:52,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:57:56,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:57:56,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:01,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:02,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:06,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:07,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:08,880 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:58:11,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:58:11,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:16,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:17,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:21,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:22,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:26,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:58:26,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:31,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:32,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:36,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:37,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:38,881 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:58:41,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:58:42,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:47,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:47,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:48,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:58:48,019 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:58:48,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:58:48,021 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:58:48,408 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:58:50,409 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:58:52,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:58:53,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:58:56,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:58:58,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:02,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:03,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:07,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:08,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:08,882 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:59:11,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:59:13,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:17,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:18,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:22,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:23,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:26,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:59:28,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:32,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:33,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:37,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:38,883 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 06:59:39,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:41,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 06:59:44,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:47,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:49,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:51,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 06:59:51,045 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 06:59:51,046 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 06:59:51,047 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 06:59:51,448 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 06:59:52,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 06:59:52,448 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 06:59:55,048 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 06:59:56,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:00:00,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:02,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:05,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:07,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:08,884 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:00:10,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:11,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:00:15,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:17,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:20,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:22,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:25,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:26,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:00:30,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:32,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:35,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:37,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:38,884 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:00:41,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:00:41,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:46,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:47,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:51,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:00:52,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:00:54,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:00:54,003 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:00:54,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:00:54,005 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:00:54,487 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:00:56,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:00:56,488 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:00:57,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:02,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:02,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:07,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:07,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:08,886 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:01:11,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:01:12,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:17,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:17,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:22,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:22,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:26,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:01:27,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:32,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:32,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:37,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:37,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:38,886 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:01:41,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:01:43,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:47,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:48,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:52,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:01:53,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:01:56,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:01:56,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:01:56,554 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:01:56,554 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:01:56,555 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:01:57,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:01:57,523 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:01:58,523 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:01:59,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:02,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:04,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:07,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:08,887 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:02:09,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:11,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:02:14,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:17,219 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:19,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:22,220 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:24,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:26,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:02:29,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:32,222 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:34,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:37,223 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:38,889 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:02:40,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:41,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:02:45,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:47,224 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:50,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:52,225 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:02:55,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:02:56,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:02:59,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:02:59,403 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:02:59,403 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:02:59,404 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:02:59,573 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:03:00,573 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:03:01,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:02,227 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:06,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:07,228 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:08,889 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:03:11,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:03:11,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:16,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:17,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:21,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:22,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:26,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:03:26,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:31,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:32,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:36,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:37,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:38,890 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:03:41,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:03:42,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:47,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:47,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:52,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:03:52,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:03:56,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:03:57,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:02,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:04:02,186 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:04:02,186 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:04:02,187 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:04:02,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:02,608 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:04:03,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:04,609 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:04:07,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:08,229 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:08,891 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:04:11,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:04:13,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:17,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:18,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:22,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:23,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:26,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:04:28,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:32,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:33,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:37,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:38,892 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:04:39,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:41,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:04:44,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:47,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:49,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:52,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:04:54,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:04:56,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:04:59,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:02,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:04,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:04,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:05:04,992 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:05:04,992 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:05:04,993 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:05:05,640 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:05:06,640 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:05:07,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:08,893 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:05:10,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:11,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:05:15,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:17,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:20,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:22,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:25,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:26,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:05:30,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:32,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:35,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:37,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:38,894 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:05:41,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:05:41,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:46,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:47,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:51,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:05:52,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:05:56,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:05:56,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:01,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:02,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:06,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:07,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:07,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:06:07,564 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:06:07,565 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:06:07,566 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:06:07,672 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:06:08,672 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:06:08,895 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:06:11,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:06:12,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:17,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:17,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:22,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:22,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:26,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:06:27,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:32,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:32,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:37,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:37,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:38,895 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:06:41,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:06:43,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:47,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:48,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:52,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:06:53,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:06:56,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:06:58,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:02,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:03,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:07,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:08,896 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:07:09,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:09,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:07:09,991 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:07:09,992 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:07:09,993 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:07:10,704 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:07:10,704 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:07:11,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:07:14,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:17,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:19,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:22,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:24,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:26,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:07:29,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:32,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:34,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:37,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:38,897 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:07:40,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:41,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:07:45,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:47,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:50,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:52,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:07:55,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:07:56,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:08:00,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:01,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:05,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:06,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:08,898 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:08:11,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:08:11,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:13,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:08:13,066 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:08:13,066 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:08:13,068 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:08:13,738 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:08:14,739 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:08:16,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:17,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:21,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:22,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:26,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:08:27,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:31,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:32,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:36,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:37,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:38,899 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:08:41,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:08:42,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:46,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:47,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:51,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:08:52,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:08:56,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:08:57,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:01,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:02,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:06,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:07,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:08,900 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:09:11,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:09:13,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:15,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:09:15,733 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:09:15,733 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:09:15,735 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:09:15,772 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:09:16,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:16,773 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:09:19,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:21,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:24,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:26,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:09:29,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:31,688 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:34,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:36,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:38,901 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:09:39,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:41,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:09:44,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:46,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:49,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:51,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:09:54,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:09:56,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:09:59,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:01,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:04,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:06,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:08,902 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:10:10,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:11,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:10:15,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:16,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:18,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:10:18,414 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:10:18,414 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:10:18,416 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:10:18,808 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:10:18,808 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:10:21,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:21,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:26,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:26,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:10:31,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:31,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:36,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:36,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:38,903 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:10:41,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:10:41,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:46,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:46,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:51,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:10:51,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:10:56,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:10:56,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:01,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:01,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:06,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:06,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:08,903 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:11:11,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:11:12,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:16,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:17,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:20,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:11:20,849 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:11:20,850 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:11:20,851 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:11:21,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:21,843 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:11:22,843 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:11:23,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:26,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:11:28,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:31,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:33,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:36,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:38,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:38,906 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:11:41,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:11:43,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:46,774 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:48,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:51,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:11:53,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:11:56,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:11:58,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:01,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:03,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:06,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:08,905 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:12:09,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:11,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:12:14,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:16,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:19,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:21,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:23,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:12:23,261 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:12:23,261 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:12:23,263 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:12:23,876 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:12:24,877 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:12:25,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:26,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:12:30,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:31,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:35,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:36,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:38,906 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:12:40,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:41,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:12:45,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:46,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:50,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:51,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:12:55,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:12:56,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:13:00,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:01,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:05,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:06,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:08,908 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:13:11,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:13:11,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:16,894 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:16,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:21,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:21,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:25,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:13:25,979 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:13:25,980 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:13:25,981 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:13:26,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:13:26,913 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:13:26,913 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:13:26,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:31,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:31,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:36,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:36,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:38,908 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:13:41,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:13:42,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:46,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:47,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:51,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:13:52,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:13:56,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:13:57,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:01,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:02,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:06,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:07,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:08,909 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:14:11,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:14:13,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:16,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:18,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:21,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:23,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:26,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:14:28,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:14:28,790 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:14:28,790 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:14:28,791 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:14:28,947 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:14:29,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:30,948 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:14:31,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:34,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:36,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:38,909 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:14:39,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:41,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:14:44,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:46,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:49,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:51,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:14:54,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:14:56,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:14:59,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:01,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:04,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:06,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:08,910 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:15:10,911 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:11,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:15:15,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:16,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:20,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:21,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:25,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:26,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:15:30,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:31,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:15:31,312 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:15:31,312 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:15:31,313 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:15:31,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:31,981 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:15:32,982 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:15:36,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:36,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:38,911 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:15:41,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:15:41,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:46,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:46,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:51,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:15:51,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:15:56,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:15:56,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:01,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:01,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:06,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:06,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:08,912 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:16:11,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:16:12,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:16,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:17,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:21,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:22,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:26,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:16:27,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:31,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:32,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:33,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:16:33,725 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:16:33,725 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:16:33,727 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:16:34,017 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:16:35,017 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:16:36,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:38,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:38,913 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:16:41,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:16:43,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:46,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:48,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:51,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:16:53,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:16:56,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:16:58,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:01,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:03,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:06,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:08,914 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:17:09,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:11,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:17:14,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:17,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:19,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:22,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:24,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:26,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:17:29,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:32,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:34,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:36,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:17:36,672 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:17:36,673 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:17:36,674 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:17:37,052 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:17:37,052 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:17:37,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:38,914 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:17:40,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:41,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:17:45,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:47,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:50,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:52,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:17:55,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:17:56,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:18:00,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:02,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:05,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:07,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:08,915 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:18:11,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:18:11,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:16,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:17,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:21,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:22,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:26,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:18:26,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:31,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:32,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:36,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:37,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:38,916 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:18:39,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:18:39,086 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:18:39,086 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:18:39,087 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:18:39,088 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:18:41,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:18:42,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:42,090 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:18:47,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:47,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:52,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:18:52,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:18:56,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:18:57,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:02,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:02,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:07,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:07,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:08,917 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:19:11,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:19:12,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:17,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:17,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:22,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:22,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:26,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:19:27,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:32,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:32,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:37,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:37,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:38,920 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:19:41,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:19:41,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:19:41,898 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:19:41,898 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:19:41,899 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:19:42,123 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:19:42,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:44,124 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:19:47,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:47,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:52,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:19:52,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:19:56,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:19:57,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:02,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:02,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:07,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:07,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:08,919 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:20:11,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:20:13,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:17,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:18,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:22,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:23,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:26,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:20:28,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:32,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:33,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:37,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:38,921 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:20:39,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:41,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:20:44,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:20:44,854 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:20:44,854 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:20:44,855 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:20:45,157 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:20:45,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:46,158 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:20:47,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:50,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:52,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:20:55,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:20:56,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:21:00,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:02,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:05,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:07,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:08,921 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:21:10,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:11,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:21:15,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:17,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:20,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:22,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:25,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:26,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:21:30,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:32,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:35,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:37,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:38,921 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:21:41,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:21:41,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:46,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:47,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:47,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:21:47,392 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:21:47,393 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:21:47,394 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:21:48,192 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:21:50,193 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:21:52,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:21:52,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:21:56,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:21:57,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:02,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:02,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:07,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:07,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:08,922 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:22:11,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:22:12,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:17,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:17,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:22,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:22,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:26,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:22:27,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:32,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:32,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:37,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:37,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:38,923 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:22:41,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:22:43,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:47,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:48,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:49,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:22:49,915 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:22:49,915 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:22:49,917 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:22:50,226 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:22:50,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:22:52,227 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:22:52,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:22:54,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:22:56,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:22:59,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:02,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:04,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:07,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:08,924 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:23:09,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:11,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:23:14,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:17,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:19,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:22,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:24,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:26,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:23:29,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:32,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:34,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:37,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:38,925 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:23:40,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:41,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:23:45,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:47,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:50,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:23:52,468 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:23:52,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:23:52,774 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:23:52,774 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:23:52,775 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:23:53,261 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:23:54,261 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:23:56,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:23:56,776 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:01,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:02,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:06,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:07,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:08,926 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:24:11,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:24:11,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:16,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:17,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:21,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:22,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:26,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:24:26,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:31,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:32,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:36,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:37,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:38,927 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:24:41,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:24:42,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:47,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:47,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:52,477 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:24:52,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:24:55,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:24:55,373 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:24:55,373 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:24:55,374 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:24:56,297 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:24:56,297 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:24:56,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:24:58,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:02,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:03,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:07,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:08,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:08,927 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:25:11,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:25:13,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:17,481 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:18,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:22,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:23,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:26,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:25:28,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:32,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:33,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:37,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:38,928 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:25:39,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:41,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:25:44,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:47,485 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:49,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:52,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:25:54,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:25:56,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:25:58,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:25:58,216 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:25:58,216 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:25:58,217 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:25:58,332 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:26:00,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:00,333 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:26:02,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:05,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:07,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:08,929 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:26:10,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:11,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:26:15,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:17,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:20,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:22,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:25,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:26,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:26:30,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:32,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:35,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:37,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:38,930 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:26:41,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:26:41,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:46,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:47,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:51,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:26:52,496 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:26:56,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:26:56,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:00,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:27:00,852 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:27:00,853 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:27:00,854 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:27:01,378 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:27:02,379 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:27:02,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:02,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:07,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:07,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:08,931 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:27:11,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:27:12,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:17,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:17,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:22,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:22,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:26,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:27:27,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:32,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:32,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:37,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:37,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:38,932 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:27:41,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:27:43,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:47,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:48,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:52,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:27:53,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:27:56,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:27:58,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:02,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:03,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:28:03,569 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:28:03,570 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:28:03,571 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:28:04,415 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:28:04,415 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:28:04,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:07,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:08,932 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:28:09,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:11,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:28:14,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:17,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:19,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:22,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:24,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:26,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:28:29,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:32,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:34,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:37,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:38,934 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:28:40,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:41,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:28:45,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:46,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:50,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:51,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:28:55,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:28:56,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:29:00,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:01,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:05,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:05,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:29:05,995 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:29:05,996 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:29:05,997 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:29:06,449 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:29:06,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:08,450 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:29:08,934 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:29:11,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:29:11,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:16,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:16,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:21,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:21,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:26,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:29:26,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:31,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:31,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:36,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:36,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:38,936 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:29:41,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:29:42,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:46,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:47,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:51,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:29:52,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:29:56,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:29:57,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:01,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:02,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:06,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:07,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:08,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:30:08,911 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:30:08,911 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:30:08,913 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:30:08,936 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:30:09,483 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:30:10,483 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:30:11,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:30:13,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:16,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:18,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:21,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:23,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:26,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:30:28,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:31,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:33,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:36,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:38,937 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:30:39,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:41,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:30:44,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:46,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:49,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:51,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:30:54,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:30:56,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:30:59,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:01,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:04,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:06,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:08,938 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:31:10,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:11,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:31:11,129 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:31:11,129 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:31:11,130 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:31:11,516 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:31:11,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:31:12,517 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:31:16,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:16,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:21,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:21,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:26,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:26,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:31:31,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:31,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:36,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:36,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:38,939 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:31:41,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:31:41,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:46,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:46,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:51,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:31:51,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:31:56,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:31:56,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:01,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:01,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:06,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:06,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:08,940 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:32:11,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:32:12,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:13,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:32:13,831 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:32:13,832 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:32:13,833 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:32:14,550 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:32:14,551 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:32:16,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:18,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:21,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:23,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:26,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:32:28,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:31,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:33,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:36,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:38,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:38,940 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:32:41,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:32:43,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:46,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:48,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:51,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:32:53,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:32:56,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:32:58,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:01,886 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:03,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:06,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:08,941 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:33:09,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:11,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:33:14,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:16,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:33:16,627 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:33:16,627 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:33:16,628 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:33:16,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:17,586 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:33:18,586 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:33:20,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:21,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:25,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:26,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:33:30,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:31,890 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:35,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:36,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:38,942 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:33:40,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:41,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:33:45,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:46,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:50,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:51,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:33:55,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:33:56,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:34:00,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:01,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:05,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:06,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:08,943 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:34:11,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:34:11,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:16,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:16,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:19,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:34:19,051 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:34:19,051 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:34:19,052 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:34:19,619 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:34:20,619 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:34:21,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:22,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:26,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:34:27,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:31,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:32,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:36,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:37,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:38,944 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:34:41,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:34:42,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:46,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:47,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:51,905 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:34:52,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:34:56,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:34:57,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:01,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:02,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:06,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:07,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:08,944 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:35:11,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:35:13,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:16,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:18,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:21,464 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:35:21,466 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:35:21,467 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:35:21,468 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:35:21,652 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:35:21,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:22,652 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:35:24,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:26,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:35:29,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:31,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:34,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:36,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:38,946 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:35:39,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:41,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:35:44,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:46,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:49,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:51,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:35:54,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:35:56,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:35:59,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:01,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:04,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:06,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:08,946 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:36:10,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:11,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:36:15,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:16,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:20,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:21,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:24,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:36:24,287 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:36:24,288 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:36:24,289 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:36:24,686 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:36:26,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:26,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:36:26,687 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:36:31,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:31,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:36,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:36,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:38,947 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:36:41,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:36:41,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:46,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:46,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:51,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:36:51,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:36:56,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:36:56,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:01,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:01,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:06,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:06,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:08,948 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:37:11,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:37:12,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:16,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:17,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:21,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:22,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:26,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:37:26,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:37:26,628 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:37:26,628 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:37:26,629 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:37:26,719 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:37:28,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:28,720 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:37:31,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:33,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:36,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:38,672 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:38,949 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:37:41,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:37:43,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:46,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:48,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:51,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:37:53,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:37:56,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:37:58,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:01,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:03,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:06,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:08,950 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:38:09,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:11,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:38:14,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:16,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:19,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:21,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:24,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:26,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:38:29,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:38:29,115 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:38:29,115 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:38:29,116 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:38:29,752 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:38:30,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:30,753 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:38:31,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:35,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:36,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:38,951 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:38:40,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:41,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:38:45,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:46,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:50,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:51,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:38:55,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:38:56,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:39:00,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:01,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:05,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:06,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:08,952 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:39:11,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:39:11,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:16,948 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:16,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:21,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:21,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:26,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:39:26,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:31,950 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:31,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:32,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:39:32,043 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:39:32,043 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:39:32,045 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:39:32,786 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:39:32,786 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:39:36,951 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:37,046 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:38,953 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:39:41,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:39:42,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:46,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:47,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:51,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:39:52,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:39:56,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:39:57,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:01,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:02,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:06,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:07,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:08,954 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:40:11,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:40:13,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:16,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:18,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:21,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:23,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:26,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:40:28,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:32,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:33,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:34,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:40:34,472 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:40:34,472 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:40:34,474 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:40:34,819 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:40:36,820 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:40:37,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:38,954 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:40:39,955 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:41,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:40:44,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:47,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:49,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:52,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:40:54,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:40:56,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:40:59,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:02,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:04,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:07,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:08,955 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:41:10,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:11,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:41:15,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:17,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:20,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:22,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:25,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:26,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:41:30,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:32,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:35,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:36,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:41:36,707 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:41:36,707 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:41:36,709 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:41:36,853 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:41:37,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:38,854 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:41:38,956 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:41:41,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:41:41,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:46,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:47,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:51,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:41:52,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:41:56,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:41:56,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:01,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:02,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:06,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:07,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:08,960 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:42:11,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:42:11,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:16,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:17,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:21,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:22,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:26,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:42:26,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:31,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:32,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:36,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:37,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:38,958 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:42:39,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:42:39,417 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:42:39,417 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:42:39,419 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:42:39,891 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:42:40,892 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:42:41,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:42:42,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:47,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:47,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:52,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:42:52,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:42:56,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:42:57,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:02,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:02,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:07,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:07,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:08,959 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:43:11,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:43:12,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:17,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:17,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:22,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:22,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:26,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:43:27,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:32,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:32,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:37,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:37,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:38,959 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:43:41,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:43:41,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:43:41,754 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:43:41,754 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:43:41,755 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:43:41,927 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:43:42,927 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:43:43,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:46,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:48,807 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:52,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:43:52,933 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:43:54,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:43:56,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:44:00,054 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:00,941 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:44:02,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:05,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:07,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:08,961 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:44:10,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:11,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:44:12,949 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:44:14,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:44:14,950 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:44:16,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:16,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:21,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:21,883 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:26,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:26,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:44:31,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:31,884 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:36,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:36,885 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:38,961 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:44:41,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:44:41,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:46,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:46,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:51,888 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:44:51,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:44:56,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:44:56,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:01,889 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:01,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:06,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:06,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:08,964 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:45:11,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:45:12,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:16,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:45:16,762 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:45:16,762 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:45:16,763 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:45:16,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:16,989 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:45:18,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:18,991 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:45:21,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:23,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:26,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:45:28,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:32,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:33,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:37,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:38,806 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:38,967 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:45:41,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:45:43,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:47,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:48,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:52,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:45:53,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:45:56,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:45:58,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:02,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:03,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:07,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:08,968 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:46:09,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:11,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:46:14,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:17,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:19,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:20,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:46:20,188 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:46:20,188 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:46:20,190 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:46:21,032 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:46:21,032 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:46:22,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:25,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:26,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:46:30,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:32,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:35,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:37,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:38,969 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:46:40,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:41,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:46:45,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:47,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:50,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:52,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:46:55,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:46:56,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:47:00,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:02,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:05,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:07,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:08,970 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:47:11,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:47:11,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:16,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:17,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:21,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:22,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:23,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:47:23,355 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:47:23,356 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:47:23,357 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:47:24,072 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:47:25,073 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:47:26,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:47:27,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:32,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:32,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:37,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:37,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:38,971 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:47:41,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:47:42,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:47,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:47,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:52,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:47:52,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:47:56,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:47:57,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:02,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:02,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:07,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:07,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:08,975 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:48:11,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:48:12,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:17,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:17,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:22,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:22,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:26,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:48:26,018 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:48:26,019 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:48:26,020 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:48:26,111 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:48:26,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:48:27,112 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:48:28,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:32,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:33,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:37,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:38,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:38,976 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:48:41,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:48:43,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:47,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:48,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:52,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:48:53,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:48:56,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:48:58,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:02,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:03,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:07,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:08,979 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:49:09,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:11,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:49:14,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:17,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:19,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:22,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:24,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:26,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:49:29,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:49:29,548 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:49:29,548 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:49:29,550 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:49:30,152 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:49:30,591 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:31,153 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:49:32,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:35,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:37,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:38,978 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:49:40,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:41,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:49:45,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:47,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:50,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:52,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:49:55,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:49:56,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:50:00,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:02,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:05,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:07,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:08,980 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:50:11,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:50:11,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:16,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:17,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:21,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:22,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:26,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:50:26,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:31,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:32,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:32,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:50:32,445 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:50:32,445 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:50:32,446 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:50:33,194 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:50:33,194 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:50:37,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:37,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:38,980 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:50:41,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:50:42,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:47,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:47,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:52,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:50:52,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:50:56,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:50:57,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:02,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:02,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:07,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:07,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:08,981 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:51:11,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:51:13,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:17,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:18,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:22,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:23,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:26,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:51:28,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:32,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:33,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:35,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:51:35,384 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:51:35,385 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:51:35,386 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:51:36,235 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:51:37,235 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:51:37,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:38,982 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:51:39,982 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:41,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:51:44,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:47,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:49,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:52,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:51:54,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:51:56,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:51:59,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:02,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:04,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:07,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:08,983 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:52:10,983 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:11,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:52:15,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:17,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:20,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:22,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:25,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:26,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:52:30,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:31,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:35,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:36,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:38,566 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:52:38,569 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:52:38,570 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:52:38,571 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:52:38,984 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:52:39,279 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:52:39,279 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:52:41,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:52:41,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:46,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:46,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:51,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:52:51,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:52:56,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:52:56,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:01,726 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:01,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:06,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:06,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:08,985 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:53:11,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:53:12,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:16,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:17,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:21,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:22,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:26,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:53:27,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:31,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:32,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:36,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:37,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:38,987 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:53:41,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:53:41,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:53:41,756 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:53:41,757 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:53:41,758 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:53:42,324 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:53:43,325 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:53:43,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:46,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:48,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:51,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:53:53,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:53:56,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:53:58,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:01,843 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:03,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:06,844 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:08,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:08,988 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:54:11,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:54:13,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:16,846 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:18,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:21,847 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:23,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:26,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:54:28,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:31,848 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:33,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:36,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:38,989 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:54:39,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:41,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:54:44,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:54:44,404 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:54:44,405 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:54:44,406 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:54:45,367 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:54:45,368 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:54:45,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:46,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:50,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:51,855 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:54:55,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:54:56,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:55:00,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:01,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:05,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:07,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:08,990 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:55:10,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:11,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:55:15,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:17,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:20,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:22,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:25,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:26,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:55:30,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:32,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:35,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:37,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:38,991 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:55:41,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:55:41,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:46,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:47,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:47,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:55:47,596 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:55:47,596 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:55:47,597 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:55:48,410 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:55:50,412 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:55:52,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:55:52,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:55:56,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:55:57,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:02,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:02,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:07,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:07,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:08,992 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:56:11,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:56:12,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:17,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:17,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:22,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:22,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:26,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:56:27,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:32,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:32,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:37,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:37,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:38,993 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:56:41,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:56:43,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:47,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:48,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:50,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:56:50,805 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:56:50,806 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:56:50,807 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:56:51,451 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:56:52,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:56:52,451 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:56:54,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:56:56,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:56:59,808 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:02,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:04,809 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:07,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:08,994 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:57:09,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:11,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:57:14,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:17,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:19,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:22,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:24,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:26,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:57:29,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:32,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:34,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:37,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:38,995 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:57:40,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:41,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:57:45,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:47,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:50,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:52,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:57:53,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:57:53,422 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:57:53,422 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:57:53,424 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:57:53,486 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:57:54,487 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:57:56,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:57:56,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:58:01,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:02,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:06,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:07,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:08,996 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:58:11,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:58:11,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:16,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:17,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:21,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:22,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:26,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:58:26,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:31,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:32,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:36,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:37,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:38,997 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:58:41,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:58:42,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:47,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:47,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:52,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:58:52,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:58:55,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:58:55,930 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:58:55,931 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:58:55,932 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:58:56,524 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 07:58:56,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:58:58,525 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 07:58:58,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:02,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:03,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:07,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:08,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:08,997 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:59:11,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:59:13,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:17,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:18,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:22,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:23,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:26,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:59:29,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:32,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:34,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:37,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:38,999 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 07:59:39,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:41,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:59:45,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:47,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:50,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:52,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 07:59:55,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 07:59:56,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 07:59:58,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 07:59:58,345 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 07:59:58,345 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 07:59:58,346 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 07:59:58,558 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:00:00,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:00,559 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:00:02,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:05,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:07,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:08,999 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:00:11,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:11,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:00:16,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:17,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:21,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:22,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:26,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:26,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:00:31,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:32,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:36,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:37,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:39,000 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:00:41,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:00:42,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:47,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:47,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:52,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:00:52,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:00:56,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:00:57,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:00,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:01:00,901 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:01:00,902 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:01:00,903 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:01:01,592 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:01:02,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:02,593 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:01:02,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:07,339 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:07,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:09,001 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:01:11,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:01:13,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:17,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:18,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:22,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:23,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:26,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:01:28,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:32,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:33,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:37,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:38,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:39,002 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:01:41,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:01:44,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:47,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:49,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:52,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:01:54,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:01:56,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:01:59,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:02,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:03,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:02:03,678 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:02:03,678 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:02:03,680 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:02:04,655 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:02:04,655 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:02:04,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:07,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:09,003 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:02:10,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:11,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:02:15,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:17,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:20,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:22,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:25,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:26,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:02:30,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:32,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:35,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:37,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:39,004 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:02:41,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:41,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:02:46,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:47,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:51,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:52,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:02:56,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:02:56,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:03:01,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:02,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:06,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:06,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:03:06,267 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:03:06,268 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:03:06,269 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:03:06,690 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:03:07,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:08,691 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:03:09,004 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:03:11,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:03:12,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:17,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:17,496 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:22,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:22,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:26,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:03:27,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:32,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:32,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:37,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:37,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:39,005 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:03:41,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:03:43,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:47,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:48,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:52,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:03:53,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:03:56,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:03:58,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:02,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:03,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:07,505 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:08,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:08,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:04:08,688 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:04:08,688 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:04:08,690 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:04:08,724 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:04:09,008 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:04:10,725 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:04:11,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:04:14,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:17,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:19,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:22,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:24,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:26,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:04:29,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:32,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:34,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:37,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:39,007 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:04:40,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:41,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:04:45,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:47,522 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:50,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:52,523 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:04:55,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:04:56,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:05:00,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:02,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:05,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:07,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:09,009 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:05:11,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:11,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:05:11,581 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:05:11,581 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:05:11,583 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:05:11,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:05:11,762 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:05:12,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:05:12,762 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:05:14,763 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:05:16,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:17,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:21,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:22,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:26,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:26,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:05:31,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:32,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:36,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:37,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:39,009 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:05:41,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:05:42,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:47,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:47,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:52,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:05:52,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:05:56,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:05:57,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:02,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:02,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:07,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:07,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:09,010 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:06:11,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:06:13,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:14,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:06:14,559 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:06:14,559 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:06:14,560 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:06:14,811 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:06:16,812 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:06:17,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:18,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:22,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:23,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:26,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:06:28,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:31,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:33,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:36,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:38,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:39,011 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:06:41,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:06:44,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:46,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:49,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:51,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:06:54,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:06:56,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:06:59,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:01,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:04,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:06,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:09,011 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:07:10,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:11,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:07:15,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:16,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:17,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:07:17,135 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:07:17,135 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:07:17,136 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:07:17,847 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:07:18,847 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:07:20,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:21,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:25,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:26,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:07:30,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:31,742 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:35,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:36,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:39,012 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:07:41,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:41,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:07:46,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:46,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:51,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:51,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:07:56,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:07:56,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:08:01,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:01,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:06,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:06,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:09,013 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:08:11,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:08:12,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:16,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:17,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:19,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:08:19,756 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:08:19,757 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:08:19,758 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:08:19,882 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:08:20,883 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:08:21,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:22,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:26,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:08:27,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:31,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:32,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:36,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:37,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:39,014 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:08:41,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:08:43,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:46,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:48,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:51,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:08:53,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:08:56,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:08:58,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:01,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:03,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:06,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:08,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:09,015 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:09:11,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:09:14,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:16,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:19,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:21,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:22,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:09:22,437 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:09:22,437 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:09:22,438 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:09:22,917 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:09:24,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:24,918 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:09:26,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:09:29,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:31,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:34,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:36,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:39,015 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:09:40,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:41,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:09:45,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:46,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:50,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:51,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:09:55,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:09:56,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:10:00,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:01,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:05,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:06,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:09,016 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:10:11,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:11,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:10:16,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:16,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:21,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:21,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:24,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:10:24,903 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:10:24,904 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:10:24,905 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:10:24,951 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:10:26,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:10:26,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:26,954 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:10:31,906 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:31,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:36,907 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:36,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:39,017 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:10:41,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:10:42,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:46,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:47,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:51,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:10:52,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:10:56,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:10:57,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:01,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:02,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:06,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:07,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:09,018 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:11:11,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:11:13,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:16,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:18,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:21,974 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:23,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:26,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:11:27,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:11:27,409 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:11:27,409 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:11:27,410 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:11:27,988 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:11:28,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:28,989 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:11:31,975 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:33,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:36,976 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:38,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:39,019 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:11:41,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:11:44,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:47,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:49,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:52,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:11:54,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:11:56,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:11:59,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:02,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:04,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:07,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:09,022 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:12:09,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:11,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:12:14,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:17,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:19,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:22,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:24,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:26,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:12:29,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:30,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:12:30,020 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:12:30,020 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:12:30,021 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:12:30,030 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:12:31,030 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:12:32,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:34,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:37,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:39,021 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:12:40,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:41,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:12:45,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:47,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:50,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:52,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:12:55,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:12:56,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:13:00,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:02,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:05,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:07,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:09,023 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:13:11,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:11,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:13:16,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:17,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:21,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:22,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:26,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:26,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:13:31,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:32,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:32,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:13:32,838 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:13:32,838 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:13:32,840 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:13:33,066 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:13:35,067 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:13:36,840 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:37,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:39,023 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:13:41,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:13:42,023 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:47,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:47,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:52,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:13:52,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:13:56,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:13:57,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:02,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:02,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:07,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:07,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:09,024 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:14:11,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:14:13,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:17,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:18,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:22,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:23,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:26,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:14:28,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:32,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:33,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:35,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:14:35,403 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:14:35,403 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:14:35,404 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:14:36,103 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:14:37,103 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:14:37,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:38,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:39,024 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:14:41,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:14:44,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:47,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:49,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:52,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:14:54,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:14:56,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:14:59,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:02,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:04,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:07,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:09,025 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:15:10,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:11,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:15:15,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:17,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:20,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:22,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:25,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:26,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:15:30,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:32,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:35,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:37,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:37,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:15:37,713 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:15:37,714 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:15:37,715 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:15:38,139 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:15:39,026 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:15:39,140 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:15:41,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:41,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:15:46,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:47,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:51,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:52,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:15:56,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:15:56,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:16:01,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:02,332 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:06,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:07,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:09,027 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:16:11,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:16:12,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:17,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:17,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:22,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:22,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:26,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:16:27,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:32,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:32,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:37,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:37,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:39,028 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:16:40,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:16:40,415 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:16:40,416 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:16:40,417 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:16:41,177 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:16:41,177 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:16:41,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:16:42,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:47,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:47,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:52,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:16:52,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:16:56,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:16:57,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:02,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:02,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:07,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:07,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:09,029 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:17:11,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:17:13,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:17,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:18,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:22,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:23,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:26,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:17:28,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:32,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:33,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:37,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:38,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:39,030 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:17:41,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:17:43,236 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:17:43,238 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:17:43,238 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:17:43,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:43,240 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:17:44,224 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:17:45,224 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:17:47,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:48,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:52,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:17:53,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:17:56,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:17:58,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:02,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:03,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:07,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:08,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:09,030 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:18:11,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:18:14,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:17,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:19,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:22,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:24,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:26,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:18:29,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:32,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:34,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:37,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:39,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:39,057 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:18:41,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:18:44,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:45,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:18:45,532 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:18:45,533 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:18:45,534 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:18:46,260 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:18:47,260 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:18:47,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:49,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:52,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:18:54,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:18:56,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:18:59,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:02,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:04,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:07,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:09,058 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:19:10,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:11,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:19:15,059 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:17,530 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:20,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:22,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:25,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:26,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:19:30,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:32,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:35,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:37,533 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:39,059 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:19:41,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:41,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:19:46,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:47,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:47,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:19:47,975 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:19:47,975 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:19:47,976 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:19:48,296 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:19:49,296 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:19:51,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:19:52,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:19:56,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:19:56,977 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:01,978 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:02,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:06,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:07,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:09,060 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:20:11,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:20:12,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:17,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:17,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:22,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:22,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:26,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:20:27,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:32,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:32,586 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:37,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:37,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:39,061 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:20:41,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:20:43,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:47,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:48,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:50,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:20:50,877 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:20:50,877 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:20:50,879 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:20:51,331 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:20:51,332 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:20:52,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:20:53,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:20:56,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:20:58,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:02,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:03,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:07,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:08,882 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:09,062 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:21:11,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:21:14,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:17,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:19,063 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:22,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:24,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:26,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:21:29,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:32,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:34,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:37,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:39,064 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:21:40,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:41,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:21:45,065 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:46,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:50,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:51,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:21:53,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:21:53,319 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:21:53,319 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:21:53,320 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:21:53,371 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:21:55,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:21:55,372 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:21:56,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:22:00,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:01,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:05,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:06,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:09,065 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:22:11,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:11,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:22:16,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:16,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:21,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:21,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:26,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:26,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:22:31,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:31,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:36,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:36,768 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:39,066 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:22:41,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:22:42,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:46,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:47,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:51,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:22:52,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:22:55,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:22:55,663 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:22:55,664 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:22:55,665 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:22:56,424 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:22:56,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:22:57,424 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:22:57,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:01,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:02,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:06,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:07,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:09,067 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:23:11,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:23:13,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:16,856 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:18,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:21,856 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:23,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:26,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:23:28,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:31,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:33,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:36,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:38,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:39,068 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:23:41,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:23:44,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:46,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:49,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:51,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:23:54,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:56,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:23:58,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:23:58,413 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:23:58,413 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:23:58,414 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:23:58,468 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:23:59,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:23:59,469 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:24:01,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:04,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:06,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:09,069 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:24:10,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:11,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:24:15,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:16,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:20,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:21,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:25,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:26,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:24:30,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:31,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:35,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:36,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:39,069 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:24:41,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:41,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:24:46,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:46,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:51,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:51,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:24:56,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:24:56,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:25:01,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:01,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:25:01,628 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:25:01,628 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:25:01,629 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:25:01,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:02,510 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:25:03,510 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:25:06,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:06,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:09,070 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:25:11,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:25:12,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:16,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:17,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:21,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:22,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:26,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:25:27,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:32,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:32,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:37,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:37,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:39,071 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:25:41,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:25:43,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:47,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:48,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:52,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:25:53,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:25:56,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:25:58,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:02,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:03,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:04,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:26:04,329 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:26:04,330 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:26:04,331 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:26:04,547 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:26:04,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:26:05,548 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:26:07,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:08,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:09,072 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:26:11,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:26:14,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:17,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:19,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:22,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:24,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:26,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:26:29,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:32,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:34,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:37,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:39,073 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:26:40,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:41,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:26:45,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:47,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:50,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:52,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:26:55,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:26:56,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:27:00,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:02,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:05,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:07,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:07,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:27:07,545 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:27:07,545 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:27:07,547 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:27:07,585 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:27:09,075 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:27:09,586 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:27:11,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:11,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:27:16,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:17,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:21,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:22,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:26,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:26,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:27:31,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:32,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:36,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:37,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:39,074 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:27:41,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:27:42,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:47,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:47,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:52,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:27:52,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:27:56,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:27:57,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:02,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:02,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:07,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:07,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:09,076 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:28:10,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:28:10,064 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:28:10,064 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:28:10,066 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:28:10,623 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:28:11,623 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:28:11,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:28:13,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:17,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:18,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:22,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:23,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:26,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:28:28,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:32,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:33,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:37,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:38,069 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:39,076 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:28:41,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:28:43,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:47,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:48,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:52,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:28:53,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:28:56,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:28:58,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:02,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:03,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:07,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:08,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:09,077 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:29:11,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:29:12,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:29:12,255 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:29:12,255 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:29:12,256 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:29:12,659 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:29:13,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:13,660 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:29:17,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:18,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:22,330 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:23,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:26,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:29:28,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:32,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:33,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:37,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:38,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:39,078 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:29:41,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:29:44,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:47,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:49,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:52,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:29:54,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:29:56,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:29:59,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:02,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:04,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:07,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:09,079 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:30:10,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:11,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:30:14,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:30:14,859 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:30:14,859 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:30:14,860 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:30:15,695 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:30:15,696 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:30:15,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:17,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:20,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:22,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:25,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:26,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:30:30,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:32,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:35,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:37,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:39,080 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:30:41,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:41,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:30:46,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:47,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:51,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:52,532 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:30:56,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:30:56,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:31:01,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:02,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:06,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:07,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:09,081 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:31:11,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:31:12,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:17,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:17,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:17,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:31:17,763 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:31:17,763 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:31:17,764 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:31:18,732 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:31:19,733 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:31:22,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:22,765 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:26,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:31:27,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:32,538 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:32,766 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:37,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:37,767 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:39,082 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:31:41,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:31:43,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:47,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:48,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:52,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:31:53,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:31:56,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:31:58,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:02,543 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:03,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:07,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:08,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:09,083 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:32:11,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:32:14,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:17,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:19,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:20,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:32:20,277 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:32:20,277 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:32:20,279 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:32:20,768 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:32:21,768 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:32:22,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:24,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:26,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:32:29,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:32,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:34,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:37,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:39,083 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:32:40,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:41,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:32:45,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:47,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:50,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:52,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:32:55,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:32:56,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:33:00,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:02,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:05,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:07,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:09,084 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:33:11,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:11,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:33:16,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:17,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:21,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:22,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:22,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:33:22,687 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:33:22,687 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:33:22,689 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:33:22,846 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:33:23,846 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:33:26,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:26,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:33:31,690 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:32,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:36,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:37,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:39,085 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:33:41,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:33:42,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:47,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:47,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:52,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:33:52,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:33:56,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:33:57,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:02,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:02,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:07,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:07,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:09,086 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:34:11,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:34:13,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:17,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:18,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:22,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:23,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:25,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:34:25,118 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:34:25,119 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:34:25,120 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:34:25,936 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:34:25,936 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:34:26,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:34:28,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:32,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:33,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:37,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:38,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:39,089 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:34:41,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:34:44,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:47,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:49,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:52,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:34:54,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:34:56,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:34:59,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:02,625 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:04,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:07,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:09,088 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:35:10,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:11,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:35:15,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:17,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:20,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:22,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:25,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:26,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:35:27,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:35:27,362 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:35:27,362 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:35:27,364 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:35:28,051 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:35:28,051 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:35:30,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:32,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:35,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:37,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:39,088 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:35:41,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:41,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:35:46,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:47,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:51,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:52,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:35:56,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:35:56,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:36:01,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:02,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:06,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:07,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:09,090 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:36:11,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:36:12,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:17,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:17,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:22,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:22,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:26,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:36:27,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:30,073 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:36:30,075 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:36:30,075 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:36:30,076 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:36:30,085 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:36:32,086 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:36:32,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:32,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:37,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:37,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:39,090 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:36:41,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:36:42,094 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:36:42,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:46,769 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:47,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:48,097 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:36:51,099 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:36:52,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:36:52,891 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:36:56,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:36:57,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:02,106 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:37:02,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:03,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:04,107 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:37:07,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:08,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:09,091 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:37:11,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:37:13,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:17,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:18,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:22,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:23,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:26,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:37:28,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:32,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:33,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:37,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:38,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:39,092 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:37:41,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:37:44,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:47,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:49,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:52,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:37:54,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:37:56,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:37:59,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:02,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:04,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:07,593 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:38:07,595 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:38:07,595 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:38:07,597 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:38:07,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:08,157 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:38:09,093 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:38:10,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:10,159 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:38:11,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:38:15,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:17,737 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:20,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:22,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:25,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:26,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:38:30,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:31,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:35,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:36,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:39,094 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:38:41,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:41,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:38:46,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:46,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:51,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:51,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:38:56,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:38:56,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:39:01,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:01,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:06,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:06,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:09,095 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:39:10,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:39:10,384 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:39:10,384 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:39:10,386 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:39:11,195 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:39:11,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:11,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:39:12,196 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:39:16,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:16,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:21,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:21,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:26,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:26,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:39:31,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:31,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:36,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:36,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:39,096 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:39:41,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:39:42,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:46,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:47,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:51,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:39:52,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:39:56,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:39:57,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:01,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:02,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:06,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:07,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:09,097 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:40:11,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:40:12,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:40:12,616 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:40:12,616 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:40:12,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:12,617 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:40:13,232 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:40:14,232 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:40:16,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:17,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:21,835 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:22,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:26,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:40:27,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:31,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:32,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:36,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:37,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:39,097 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:40:41,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:40:43,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:46,839 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:48,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:51,840 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:40:53,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:40:56,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:40:58,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:01,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:03,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:06,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:08,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:09,098 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:41:11,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:41:14,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:15,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:41:15,350 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:41:15,350 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:41:15,352 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:41:16,269 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:41:16,269 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:41:16,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:19,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:21,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:24,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:26,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:41:29,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:31,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:34,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:36,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:39,099 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:41:40,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:41,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:41:45,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:46,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:50,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:51,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:41:55,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:41:56,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:42:00,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:01,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:05,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:06,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:09,103 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:42:10,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:11,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:42:15,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:16,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:18,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:42:18,252 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:42:18,252 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:42:18,254 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:42:18,305 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:42:20,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:21,307 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:42:21,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:25,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:26,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:42:30,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:31,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:35,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:36,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:39,101 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:42:41,101 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:41,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:42:46,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:46,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:51,102 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:51,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:42:56,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:42:56,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:43:01,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:02,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:06,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:07,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:09,102 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:43:11,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:43:12,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:17,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:17,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:21,061 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:43:21,064 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:43:21,064 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:43:21,065 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:43:21,342 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:43:22,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:23,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:23,343 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:43:26,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:43:28,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:32,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:33,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:37,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:38,068 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:39,102 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:43:41,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:43:43,103 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:47,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:48,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:52,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:43:53,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:43:56,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:43:58,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:02,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:03,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:07,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:08,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:09,103 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:44:11,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:44:14,104 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:17,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:19,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:22,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:23,814 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:44:23,816 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:44:23,816 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:44:23,818 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:44:24,379 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:44:24,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:25,380 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:44:26,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:44:29,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:32,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:34,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:37,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:39,104 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:44:40,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:41,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:44:45,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:47,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:50,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:52,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:44:55,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:44:56,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:45:00,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:02,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:05,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:07,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:09,105 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:45:11,105 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:11,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:45:16,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:17,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:21,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:22,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:26,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:26,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:45:26,725 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:45:26,725 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:45:26,727 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:45:26,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:45:27,418 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:45:29,420 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:45:31,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:32,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:36,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:37,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:39,106 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:45:41,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:45:42,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:47,107 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:47,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:52,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:45:52,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:45:56,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:45:57,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:02,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:02,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:07,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:07,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:09,107 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:46:11,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:46:13,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:17,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:18,108 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:22,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:23,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:26,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:46:28,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:29,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:46:29,523 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:46:29,523 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:46:29,525 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:46:30,470 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:46:31,471 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:46:32,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:33,564 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:37,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:38,565 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:39,108 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:46:41,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:46:44,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:47,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:49,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:52,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:46:54,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:46:56,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:46:59,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:02,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:04,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:07,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:09,109 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:47:10,109 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:11,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:47:15,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:17,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:20,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:22,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:25,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:26,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:47:30,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:32,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:47:32,443 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:47:32,443 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:47:32,445 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:47:32,550 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:47:32,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:32,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:47:35,552 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:47:35,946 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:37,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:39,110 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:47:41,110 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:41,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:47:46,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:47,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:51,111 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:52,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:47:56,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:47:56,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:48:01,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:02,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:06,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:07,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:09,111 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:48:11,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:48:12,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:17,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:17,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:22,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:22,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:26,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:48:27,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:32,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:32,642 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:35,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:48:35,511 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:48:35,512 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:48:35,513 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:48:35,637 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:48:37,513 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:37,638 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:48:37,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:39,112 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:48:41,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:48:43,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:47,645 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:48,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:52,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:48:53,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:48:56,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:48:58,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:02,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:03,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:07,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:08,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:09,113 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:49:11,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:49:14,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:17,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:19,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:22,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:24,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:26,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:49:29,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:32,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:34,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:37,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:37,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:49:37,972 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:49:37,973 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:49:37,974 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:49:38,677 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:49:39,113 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:49:39,678 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:49:40,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:41,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:49:45,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:47,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:50,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:52,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:49:55,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:49:56,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:50:00,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:02,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:05,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:07,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:09,114 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:50:11,115 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:11,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:50:16,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:17,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:21,116 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:22,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:26,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:26,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:50:31,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:32,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:36,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:37,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:39,115 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:50:40,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:50:40,003 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:50:40,004 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:50:40,005 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:50:40,714 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:50:41,715 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:50:41,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:50:42,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:47,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:47,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:52,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:50:52,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:50:56,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:50:57,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:02,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:02,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:07,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:07,752 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:09,116 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:51:11,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:51:12,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:17,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:17,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:22,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:22,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:26,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:51:27,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:32,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:32,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:37,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:37,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:39,117 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:51:41,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:51:42,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:51:42,636 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:51:42,637 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:51:42,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:42,638 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:51:42,750 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:51:45,752 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:51:47,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:47,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:52,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:51:52,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:51:56,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:51:57,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:01,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:02,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:06,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:07,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:09,118 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:52:11,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:52:13,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:16,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:18,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:21,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:23,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:26,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:52:28,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:31,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:33,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:36,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:38,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:39,119 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:52:41,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:52:44,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:45,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:52:45,311 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:52:45,311 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:52:45,313 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:52:45,787 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:52:46,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:47,788 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:52:49,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:51,855 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:52:54,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:52:56,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:52:59,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:01,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:04,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:06,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:09,119 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:53:10,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:11,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:53:15,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:16,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:20,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:21,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:25,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:26,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:53:30,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:31,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:35,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:36,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:39,120 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:53:41,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:41,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:53:46,121 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:46,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:47,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:53:47,564 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:53:47,566 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:53:47,567 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:53:47,823 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:53:49,824 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:53:51,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:51,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:53:56,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:53:56,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:54:01,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:01,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:06,569 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:06,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:09,121 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:54:11,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:54:12,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:16,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:17,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:21,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:22,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:26,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:54:27,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:31,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:32,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:36,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:37,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:39,122 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:54:41,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:54:43,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:46,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:48,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:50,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:54:50,234 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:54:50,235 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:54:50,236 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:54:50,858 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:54:51,858 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:54:51,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:54:53,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:54:56,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:54:58,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:01,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:03,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:06,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:08,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:09,123 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:55:11,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:55:14,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:16,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:19,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:21,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:24,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:26,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:55:29,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:31,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:34,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:36,913 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:39,124 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:55:40,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:41,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:55:45,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:46,914 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:50,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:51,915 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:55:52,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:55:52,609 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:55:52,609 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:55:52,610 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:55:52,893 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:55:53,893 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:55:55,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:55:56,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:56:00,611 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:01,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:05,612 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:06,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:09,125 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:56:11,125 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:11,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:56:16,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:16,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:21,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:21,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:26,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:26,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:56:31,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:31,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:36,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:36,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:39,126 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:56:41,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:56:42,126 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:46,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:47,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:51,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:56:52,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:56:54,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:56:54,547 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:56:54,547 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:56:54,549 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:56:54,927 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:56:55,928 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:56:56,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:56:57,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:01,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:02,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:06,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:07,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:09,129 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:57:11,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:57:13,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:16,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:18,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:21,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:23,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:26,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:57:28,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:31,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:33,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:36,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:38,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:39,127 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:57:41,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:57:44,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:47,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:49,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:52,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:57:54,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:57:56,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:57:57,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:57:57,031 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:57:57,031 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:57:57,033 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:57:57,964 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:57:59,965 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:58:00,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:02,050 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:05,071 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:07,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:09,128 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:58:10,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:11,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:58:15,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:17,230 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:20,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:22,231 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:25,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:26,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:58:30,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:32,232 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:35,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:37,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:39,129 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:58:41,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:41,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:58:46,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:47,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:51,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:52,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:58:56,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:58:56,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:58:59,949 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 08:58:59,951 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 08:58:59,951 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 08:58:59,953 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 08:58:59,999 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 08:59:01,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:02,000 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 08:59:02,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:06,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:07,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:09,130 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:59:11,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:59:12,130 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:17,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:17,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:22,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:22,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:26,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:59:27,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:32,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:32,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:37,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:37,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:39,131 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 08:59:41,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:59:43,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:47,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:48,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:52,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 08:59:53,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 08:59:56,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 08:59:58,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:02,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:02,520 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:00:02,522 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:00:02,522 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:00:02,524 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:00:03,036 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:00:03,524 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:04,036 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:00:07,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:08,525 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:09,131 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:00:11,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:00:14,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:17,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:19,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:22,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:24,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:26,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:00:29,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:32,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:34,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:37,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:39,132 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:00:40,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:41,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:00:45,133 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:47,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:50,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:52,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:00:55,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:00:56,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:01:00,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:02,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:04,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:01:04,497 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:01:04,497 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:01:04,498 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:01:05,099 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:01:05,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:06,100 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:01:07,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:09,133 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:01:11,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:11,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:01:16,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:17,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:21,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:22,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:26,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:26,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:01:31,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:32,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:36,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:37,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:39,134 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:01:41,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:01:42,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:47,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:47,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:52,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:01:52,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:01:56,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:01:57,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:02,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:02,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:07,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:07,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:02:07,293 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:02:07,293 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:02:07,295 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:02:07,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:08,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:02:09,134 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:02:10,137 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:02:11,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:02:13,135 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:17,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:18,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:22,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:23,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:26,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:02:28,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:32,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:33,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:37,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:38,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:39,136 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:02:41,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:02:44,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:47,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:49,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:52,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:02:54,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:02:56,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:02:59,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:02,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:04,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:07,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:09,137 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:03:09,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:03:09,746 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:03:09,746 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:03:09,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:09,748 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:03:10,172 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:03:11,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:03:12,173 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:03:14,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:17,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:19,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:22,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:24,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:26,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:03:29,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:32,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:34,751 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:37,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:39,138 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:03:40,138 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:41,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:03:45,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:47,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:50,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:52,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:03:55,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:03:56,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:04:00,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:02,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:05,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:07,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:09,139 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:04:11,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:11,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:04:12,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:04:12,164 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:04:12,165 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:04:12,166 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:04:12,208 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:04:14,209 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:04:16,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:17,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:21,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:22,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:26,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:26,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:04:31,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:32,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:36,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:37,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:39,139 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:04:41,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:04:42,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:47,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:47,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:52,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:04:52,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:04:56,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:04:57,142 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:02,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:02,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:07,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:07,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:09,143 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:05:11,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:05:12,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:14,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:05:14,712 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:05:14,712 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:05:14,714 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:05:15,244 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:05:16,244 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:05:17,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:17,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:22,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:22,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:26,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:05:27,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:32,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:32,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:37,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:37,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:39,141 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:05:41,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:05:43,142 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:47,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:48,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:52,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:05:53,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:05:56,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:05:58,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:02,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:03,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:07,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:08,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:09,143 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:06:11,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:06:14,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:17,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:06:17,446 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:06:17,447 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:06:17,448 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:06:17,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:18,281 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:06:19,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:20,282 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:06:22,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:24,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:26,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:06:29,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:32,488 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:34,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:37,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:39,143 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:06:40,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:41,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:06:45,144 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:47,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:50,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:52,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:06:55,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:06:56,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:07:00,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:02,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:05,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:07,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:09,144 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:07:11,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:11,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:07:16,145 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:17,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:19,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:07:19,627 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:07:19,627 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:07:19,629 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:07:20,316 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:07:21,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:22,317 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:07:22,496 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:26,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:26,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:07:31,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:32,497 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:36,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:37,498 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:39,145 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:07:41,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:07:42,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:47,146 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:47,499 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:52,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:07:52,500 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:07:56,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:07:57,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:02,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:02,501 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:07,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:07,502 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:09,146 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:08:11,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:08:13,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:17,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:18,147 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:22,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:08:22,478 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:08:22,479 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:08:22,480 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:08:22,504 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:22,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:08:23,351 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:08:23,979 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:24,352 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:08:26,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:08:28,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:32,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:33,980 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:37,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:38,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:39,147 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:08:41,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:08:44,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:47,508 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:49,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:52,509 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:08:54,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:08:56,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:08:59,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:02,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:04,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:07,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:09,148 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:09:10,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:11,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:09:15,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:17,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:20,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:22,598 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:25,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:25,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:09:25,866 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:09:25,866 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:09:25,867 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:09:26,399 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:09:26,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:09:28,400 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:09:30,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:32,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:35,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:37,600 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:39,149 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:09:41,149 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:41,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:09:46,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:47,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:51,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:52,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:09:56,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:09:56,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:10:01,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:02,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:06,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:07,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:09,150 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:10:11,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:10:12,150 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:17,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:17,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:22,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:22,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:26,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:10:27,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:28,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:10:28,575 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:10:28,575 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:10:28,576 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:10:29,447 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:10:30,447 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:10:32,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:32,640 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:37,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:37,641 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:39,150 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:10:41,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:10:43,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:47,643 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:48,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:52,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:10:53,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:10:56,824 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:10:58,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:02,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:03,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:07,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:08,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:09,151 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:11:11,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:11:14,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:17,648 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:19,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:22,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:24,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:26,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:11:29,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:31,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:11:31,196 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:11:31,197 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:11:31,198 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:11:31,487 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:11:32,488 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:11:32,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:34,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:37,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:39,152 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:11:40,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:41,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:11:45,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:47,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:50,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:52,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:11:55,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:11:56,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:12:00,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:02,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:05,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:07,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:09,153 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:12:11,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:11,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:12:16,154 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:17,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:21,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:22,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:26,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:26,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:12:31,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:32,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:33,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:12:33,856 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:12:33,857 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:12:33,859 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:12:34,524 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:12:36,525 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:12:36,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:37,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:39,154 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:12:41,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:12:42,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:47,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:47,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:52,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:12:52,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:12:56,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:12:57,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:02,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:02,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:07,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:07,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:09,155 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:13:11,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:13:13,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:17,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:18,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:22,682 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:23,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:26,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:13:28,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:32,684 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:33,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:36,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:13:36,441 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:13:36,441 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:13:36,442 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:13:36,560 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:13:37,685 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:38,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:38,561 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:13:39,156 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:13:41,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:13:44,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:47,686 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:49,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:52,687 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:13:54,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:13:56,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:13:59,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:02,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:04,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:07,689 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:09,157 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:14:10,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:11,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:14:15,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:17,691 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:20,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:22,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:25,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:26,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:14:30,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:32,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:35,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:37,755 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:39,148 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:14:39,150 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:14:39,151 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:14:39,152 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:14:39,157 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:14:39,596 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:14:40,596 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:14:41,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:41,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:14:46,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:47,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:51,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:52,758 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:14:56,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:14:56,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:15:01,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:02,759 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:06,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:07,816 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:09,158 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:15:11,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:15:12,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:17,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:17,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:22,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:22,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:26,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:15:27,160 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:32,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:32,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:37,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:37,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:39,159 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:15:41,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:15:41,411 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:15:41,411 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:15:41,412 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:15:41,631 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:15:41,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:15:42,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:42,632 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:15:46,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:47,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:51,902 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:15:52,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:15:56,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:15:57,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:01,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:02,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:06,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:07,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:09,160 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:16:11,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:16:13,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:16,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:18,161 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:21,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:23,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:26,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:16:28,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:32,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:33,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:37,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:38,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:39,161 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:16:41,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:16:44,162 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:44,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:16:44,222 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:16:44,223 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:16:44,224 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:16:44,667 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:16:46,668 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:16:47,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:49,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:52,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:16:54,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:16:56,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:16:59,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:02,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:04,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:07,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:09,162 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:17:10,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:11,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:17:15,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:17,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:20,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:22,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:25,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:26,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:17:30,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:32,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:35,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:37,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:39,162 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:17:41,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:41,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:17:46,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:47,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:17:47,003 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:17:47,003 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:17:47,005 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:17:47,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:47,703 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:17:48,704 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:17:52,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:17:52,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:17:56,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:17:57,044 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:02,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:02,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:07,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:07,045 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:09,163 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:18:11,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:18:12,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:17,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:17,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:22,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:22,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:26,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:18:27,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:32,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:32,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:37,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:37,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:39,164 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:18:41,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:18:43,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:47,020 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:48,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:49,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:18:49,235 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:18:49,235 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:18:49,236 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:18:49,739 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:18:50,739 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:18:52,021 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:18:53,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:18:56,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:18:58,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:02,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:03,238 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:07,022 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:08,239 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:09,165 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:19:11,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:19:14,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:17,024 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:19,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:22,025 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:24,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:26,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:19:29,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:32,026 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:34,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:37,027 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:39,166 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:19:40,166 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:41,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:19:45,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:47,028 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:50,167 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:51,916 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:19:51,918 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:19:51,919 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:19:51,920 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:19:52,029 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:19:52,775 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:19:52,775 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:19:55,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:19:56,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:20:00,921 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:02,030 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:05,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:07,031 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:09,169 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:20:11,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:11,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:20:16,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:17,032 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:21,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:22,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:26,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:26,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:20:31,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:32,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:36,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:37,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:39,168 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:20:41,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:20:42,168 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:47,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:47,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:52,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:20:52,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:20:54,562 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:20:54,564 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:20:54,564 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:20:54,566 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:20:54,812 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:20:56,813 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:20:56,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:20:57,567 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:02,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:02,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:07,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:07,568 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:09,169 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:21:11,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:21:13,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:17,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:18,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:22,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:23,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:26,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:21:28,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:32,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:33,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:37,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:38,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:39,170 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:21:41,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:21:44,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:47,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:49,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:52,091 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:21:54,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:21:56,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:21:57,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:21:57,036 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:21:57,036 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:21:57,037 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:21:57,860 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:21:58,860 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:22:00,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:02,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:05,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:07,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:09,170 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:22:10,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:11,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:22:15,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:17,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:20,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:22,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:25,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:26,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:22:30,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:32,097 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:35,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:37,098 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:39,171 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:22:41,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:41,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:22:46,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:47,099 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:51,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:52,100 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:22:56,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:22:56,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:22:59,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:22:59,552 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:22:59,552 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:22:59,553 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:22:59,897 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:23:00,897 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:23:01,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:02,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:06,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:07,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:09,172 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:23:11,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:23:12,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:17,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:17,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:22,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:22,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:26,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:23:27,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:32,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:32,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:37,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:37,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:39,173 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:23:41,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:23:43,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:47,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:48,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:52,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:23:53,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:23:56,829 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:23:58,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:01,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:24:01,854 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:24:01,855 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:24:01,856 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:24:01,935 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:24:02,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:02,935 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:24:03,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:07,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:08,895 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:09,174 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:24:11,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:24:14,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:17,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:19,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:22,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:24,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:26,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:24:29,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:32,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:34,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:37,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:39,175 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:24:40,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:41,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:24:45,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:47,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:50,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:52,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:24:55,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:24:56,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:25:00,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:02,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:04,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:25:04,381 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:25:04,382 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:25:04,383 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:25:04,972 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:25:04,972 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:25:05,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:07,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:09,176 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:25:11,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:11,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:25:16,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:17,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:21,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:22,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:26,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:26,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:25:31,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:32,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:36,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:37,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:39,177 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:25:41,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:25:42,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:47,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:47,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:52,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:25:52,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:25:56,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:25:57,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:02,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:02,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:06,981 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:26:06,984 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:26:06,984 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:26:06,986 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:26:07,012 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:26:07,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:07,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:09,178 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:26:10,014 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:26:11,830 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:26:13,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:17,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:18,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:22,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:23,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:26,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:26:28,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:32,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:33,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:37,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:38,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:39,179 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:26:41,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:26:44,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:47,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:49,180 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:52,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:26:54,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:26:56,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:26:59,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:02,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:04,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:07,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:09,179 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:27:09,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:27:09,199 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:27:09,200 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:27:09,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:09,201 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:27:10,049 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:27:11,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:27:12,050 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:27:14,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:17,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:19,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:22,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:24,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:26,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:27:29,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:32,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:34,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:37,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:39,182 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:27:40,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:41,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:27:45,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:47,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:50,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:52,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:27:55,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:27:56,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:28:00,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:02,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:05,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:07,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:09,181 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:28:11,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:11,741 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:28:11,743 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:28:11,744 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:28:11,746 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:28:11,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:28:12,097 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:28:14,099 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:28:16,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:17,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:21,747 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:22,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:26,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:26,831 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:28:31,748 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:32,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:36,749 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:37,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:39,182 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:28:41,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:28:42,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:47,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:47,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:52,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:28:52,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:28:56,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:28:57,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:02,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:02,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:07,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:07,322 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:09,183 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:29:11,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:29:13,184 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:14,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:29:14,530 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:29:14,531 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:29:14,532 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:29:15,134 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:29:16,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:29:17,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:18,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:22,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:23,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:26,141 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:29:26,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:29:29,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:32,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:34,146 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:29:34,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:37,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:39,184 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:29:40,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:42,594 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:29:45,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:47,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:29:48,174 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:29:48,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:50,176 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:29:51,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:53,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:29:56,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:29:56,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:30:01,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:02,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:06,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:07,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:09,185 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:30:11,863 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:30:12,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:17,186 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:17,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:22,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:22,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:26,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:30:27,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:32,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:32,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:37,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:37,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:39,186 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:30:41,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:30:43,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:47,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:48,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:50,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:30:50,211 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:30:50,212 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:30:50,213 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:30:50,218 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:30:52,219 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:30:52,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:30:53,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:30:56,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:30:58,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:02,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:03,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:07,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:08,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:09,187 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:31:11,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:31:14,187 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:17,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:19,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:22,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:24,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:26,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:31:29,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:32,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:34,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:37,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:39,188 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:31:40,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:41,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:31:45,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:47,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:50,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:52,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:31:52,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:31:52,537 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:31:52,538 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:31:52,539 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:31:53,293 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:31:54,294 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:31:55,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:31:56,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:32:00,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:02,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:05,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:07,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:09,188 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:32:11,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:11,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:32:16,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:17,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:21,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:22,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:26,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:26,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:32:31,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:32,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:36,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:37,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:39,189 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:32:41,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:32:42,190 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:47,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:47,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:52,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:52,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:32:55,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:32:55,036 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:32:55,037 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:32:55,038 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:32:55,334 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:32:56,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:32:58,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:32:58,336 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:33:02,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:03,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:07,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:08,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:09,190 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:33:11,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:33:13,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:17,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:18,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:22,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:23,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:26,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:33:28,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:32,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:33,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:37,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:38,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:39,191 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:33:41,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:33:44,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:47,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:49,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:52,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:33:54,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:33:56,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:33:57,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:33:57,434 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:33:57,434 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:33:57,435 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:33:58,373 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:33:59,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:00,374 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:34:02,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:04,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:07,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:09,192 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:34:10,192 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:11,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:34:15,193 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:17,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:20,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:22,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:25,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:26,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:34:30,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:32,428 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:35,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:37,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:39,195 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:34:40,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:41,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:34:45,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:47,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:50,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:52,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:34:55,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:34:56,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:35:00,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:35:00,158 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:35:00,158 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:35:00,160 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:35:00,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:00,411 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:35:02,412 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:35:02,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:05,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:07,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:09,194 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:35:11,194 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:11,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:35:16,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:17,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:21,195 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:22,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:26,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:26,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:35:31,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:32,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:36,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:37,438 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:39,195 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:35:41,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:35:42,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:47,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:47,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:52,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:35:52,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:35:56,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:35:57,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:02,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:02,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:02,887 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:36:02,889 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:36:02,890 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:36:02,892 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:36:03,448 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:36:04,449 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:36:07,514 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:07,893 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:09,195 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:36:11,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:36:13,196 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:17,516 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:18,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:22,517 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:23,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:26,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:36:28,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:32,577 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:33,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:37,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:38,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:39,196 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:36:41,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:36:44,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:47,580 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:49,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:52,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:36:54,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:36:56,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:36:59,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:02,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:04,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:05,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:37:05,273 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:37:05,273 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:37:05,276 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:37:05,485 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:37:07,583 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:08,486 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:37:09,197 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:37:10,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:11,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:37:15,198 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:17,584 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:20,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:22,585 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:25,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:26,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:37:30,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:32,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:35,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:37,647 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:39,198 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:37:41,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:41,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:37:46,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:47,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:51,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:52,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:37:56,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:37:56,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:38:01,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:02,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:06,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:07,652 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:07,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:38:07,724 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:38:07,724 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:38:07,726 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:38:08,521 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:38:09,199 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:38:10,522 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:38:11,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:38:12,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:17,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:17,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:22,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:22,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:26,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:38:27,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:32,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:32,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:37,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:37,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:39,200 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:38:41,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:38:43,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:47,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:48,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:52,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:38:53,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:38:56,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:38:58,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:02,706 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:03,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:07,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:08,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:09,201 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:39:10,201 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:39:10,203 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:39:10,203 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:39:10,205 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:39:10,557 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:39:11,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:39:12,558 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:39:13,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:17,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:18,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:22,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:23,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:26,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:39:28,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:32,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:33,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:37,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:38,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:39,202 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:39:41,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:39:44,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:47,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:49,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:52,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:39:54,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:39:56,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:39:59,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:02,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:04,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:07,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:09,203 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:40:10,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:11,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:40:12,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:40:12,945 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:40:12,945 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:40:12,946 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:40:13,594 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:40:14,594 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:40:15,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:17,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:20,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:22,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:25,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:26,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:40:30,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:32,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:35,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:37,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:39,204 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:40:41,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:41,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:40:46,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:47,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:51,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:52,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:40:56,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:40:56,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:41:01,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:02,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:06,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:07,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:09,205 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:41:11,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:41:12,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:15,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:41:15,553 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:41:15,553 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:41:15,554 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:41:15,628 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:41:17,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:17,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:18,630 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:41:22,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:22,819 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:26,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:41:27,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:31,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:32,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:36,880 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:37,597 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:39,206 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:41:41,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:41:43,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:46,881 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:48,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:51,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:41:53,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:41:56,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:41:58,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:01,922 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:03,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:06,923 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:08,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:09,207 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:42:11,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:42:14,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:16,924 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:18,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:42:18,261 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:42:18,261 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:42:18,262 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:42:18,662 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:42:19,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:20,672 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:42:21,925 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:24,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:26,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:42:29,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:31,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:34,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:36,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:39,207 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:42:40,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:41,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:42:45,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:46,928 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:50,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:51,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:42:55,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:42:56,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:43:00,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:01,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:05,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:06,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:09,208 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:43:11,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:11,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:43:16,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:16,933 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:20,753 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:43:20,758 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:43:20,759 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:43:20,762 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:43:21,706 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:43:21,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:21,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:22,707 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:43:26,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:26,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:43:31,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:31,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:36,764 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:36,936 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:39,209 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:43:41,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:43:42,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:46,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:47,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:51,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:43:52,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:43:56,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:43:57,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:01,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:02,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:06,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:07,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:09,210 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:44:11,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:44:13,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:16,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:18,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:21,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:23,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:23,521 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:44:23,523 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:44:23,523 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:44:23,525 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:44:23,745 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:44:24,746 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:44:26,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:44:28,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:31,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:33,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:36,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:38,527 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:39,211 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:44:41,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:44:44,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:47,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:49,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:52,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:44:54,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:44:56,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:44:59,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:02,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:04,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:07,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:09,212 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:45:10,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:11,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:45:15,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:17,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:20,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:22,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:25,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:25,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:45:25,488 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:45:25,488 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:45:25,490 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:45:25,781 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:45:26,782 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:45:26,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:45:30,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:32,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:35,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:37,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:39,212 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:45:41,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:41,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:45:46,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:47,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:51,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:52,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:45:56,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:45:56,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:46:01,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:02,019 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:06,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:07,072 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:09,213 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:46:11,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:46:12,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:17,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:17,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:22,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:22,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:26,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:46:27,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:28,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:46:28,390 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:46:28,390 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:46:28,391 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:46:28,818 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:46:30,819 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:46:32,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:32,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:37,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:37,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:39,214 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:46:41,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:46:43,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:47,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:48,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:52,081 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:46:53,216 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:46:56,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:46:58,217 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:02,082 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:03,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:07,083 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:08,218 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:09,241 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:47:11,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:47:13,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:17,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:18,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:22,086 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:23,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:26,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:47:28,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:31,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:47:31,217 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:47:31,217 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:47:31,218 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:47:31,854 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:47:32,087 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:32,854 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:47:33,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:37,088 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:38,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:39,242 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:47:41,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:47:44,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:47,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:49,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:52,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:47:54,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:47:56,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:47:59,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:02,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:04,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:07,092 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:09,243 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:48:10,243 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:11,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:48:15,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:17,093 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:20,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:22,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:25,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:26,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:48:30,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:32,095 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:33,237 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:48:33,239 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:48:33,240 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:48:33,241 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:48:33,888 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:48:34,888 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:48:36,242 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:37,096 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:39,244 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:48:41,244 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:41,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:48:46,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:47,169 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:51,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:52,170 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:48:56,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:48:56,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:49:01,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:02,171 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:06,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:07,172 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:09,245 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:49:11,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:49:12,245 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:17,173 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:17,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:22,174 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:22,246 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:26,871 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:49:27,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:32,175 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:32,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:35,832 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:49:35,834 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:49:35,835 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:49:35,836 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:49:35,923 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:49:36,923 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:49:37,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:37,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:39,248 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:49:41,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:49:43,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:47,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:48,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:52,178 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:49:53,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:49:56,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:49:58,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:02,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:03,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:07,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:08,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:09,246 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:50:11,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:50:14,247 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:17,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:19,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:22,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:24,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:26,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:50:29,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:32,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:34,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:37,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:38,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:50:38,474 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:50:38,474 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:50:38,476 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:50:38,959 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:50:39,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:50:39,247 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:50:40,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:40,961 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:50:41,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:50:45,248 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:47,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:50,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:52,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:50:55,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:50:56,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:51:00,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:02,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:05,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:07,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:09,248 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:51:11,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:11,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:51:16,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:17,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:21,250 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:22,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:26,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:26,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:51:31,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:32,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:36,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:37,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:39,249 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:51:41,596 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:51:41,598 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:51:41,598 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:51:41,599 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:41,600 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:51:41,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:51:41,996 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:51:42,997 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:51:46,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:47,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:51,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:52,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:51:56,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:51:56,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:52:01,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:02,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:06,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:07,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:09,250 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:52:11,872 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:52:12,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:17,251 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:17,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:22,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:22,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:26,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:52:27,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:32,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:32,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:37,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:37,323 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:39,251 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:52:41,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:52:43,252 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:43,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:52:43,974 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:52:43,974 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:52:43,975 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:52:44,044 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:52:45,044 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:52:47,324 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:49,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:52,325 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:52:54,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:52:56,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:52:59,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:02,333 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:04,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:07,334 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:09,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:09,252 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:53:11,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:53:14,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:17,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:19,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:22,336 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:24,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:26,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:53:29,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:32,337 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:34,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:37,338 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:39,253 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:53:40,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:41,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:53:45,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:46,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:53:46,368 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:53:46,368 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:53:46,369 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:53:47,081 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:53:47,340 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:49,082 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:53:50,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:52,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:53:55,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:53:56,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:54:00,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:02,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:05,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:07,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:09,253 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:54:11,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:11,873 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:54:16,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:17,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:21,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:22,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:26,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:26,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:54:31,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:32,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:36,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:37,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:39,254 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:54:41,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:54:42,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:47,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:47,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:48,757 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:54:48,759 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:54:48,759 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:54:48,761 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:54:49,117 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:54:51,119 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:54:52,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:54:52,761 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:54:56,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:54:57,762 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:02,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:02,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:07,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:07,763 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:09,255 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:55:11,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:55:13,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:17,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:18,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:22,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:23,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:26,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:55:28,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:32,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:33,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:37,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:38,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:39,256 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:55:41,874 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:55:44,256 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:47,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:49,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:50,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:55:50,840 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:55:50,840 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:55:50,842 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:55:51,193 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:55:52,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:55:53,194 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:55:54,843 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:55:56,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:55:59,843 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:02,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:04,844 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:07,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:09,257 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:56:10,257 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:11,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:56:15,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:17,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:20,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:22,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:25,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:26,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:56:30,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:32,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:35,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:37,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:39,257 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:56:41,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:41,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:56:46,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:47,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:51,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:52,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:56:53,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:56:53,287 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:56:53,288 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:56:53,289 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:56:54,233 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:56:55,233 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:56:56,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:56:56,876 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:57:01,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:02,454 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:06,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:07,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:09,259 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:57:11,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:57:12,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:17,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:17,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:22,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:22,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:26,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:57:27,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:32,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:32,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:37,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:37,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:39,263 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:57:41,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:57:43,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:47,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:48,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:52,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:57:53,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:57:55,892 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:57:55,894 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:57:55,894 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:57:55,895 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:57:56,279 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:57:56,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:57:57,279 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:57:58,896 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:02,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:03,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:07,464 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:08,897 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:09,260 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:58:11,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:58:14,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:17,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:19,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:22,466 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:24,262 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:26,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:58:29,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:32,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:34,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:37,468 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:39,262 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:58:40,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:41,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:58:45,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:47,470 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:50,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:52,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:58:55,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:58:56,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:58:58,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 09:58:58,193 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 09:58:58,193 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 09:58:58,194 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 09:58:58,316 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 09:58:59,317 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 09:59:01,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:02,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:06,235 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:07,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:09,262 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:59:11,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:11,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:59:16,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:17,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:21,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:22,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:26,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:26,877 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:59:31,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:32,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:36,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:37,477 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:39,263 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 09:59:41,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:59:42,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:47,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:47,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:52,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 09:59:52,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 09:59:56,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 09:59:57,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:00,770 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:00:00,772 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:00:00,772 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:00:00,773 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:00:01,352 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:00:02,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:02,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:03,354 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:00:07,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:07,815 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:09,264 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:00:11,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:00:13,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:17,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:18,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:22,485 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:23,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:26,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:00:28,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:32,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:33,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:37,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:38,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:39,265 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:00:41,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:00:44,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:47,554 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:49,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:52,555 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:00:54,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:00:56,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:00:59,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:02,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:03,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:01:03,087 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:01:03,087 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:01:03,088 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:01:03,388 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:01:05,089 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:05,390 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:01:07,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:09,265 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:01:10,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:11,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:01:15,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:17,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:20,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:22,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:25,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:26,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:01:30,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:32,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:35,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:37,561 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:39,266 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:01:41,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:41,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:01:46,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:47,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:51,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:52,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:01:56,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:01:56,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:02:01,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:02,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:05,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:02:05,301 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:02:05,302 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:02:05,303 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:02:05,426 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:02:06,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:07,428 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:02:07,653 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:09,267 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:02:11,878 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:02:12,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:17,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:17,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:22,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:22,655 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:26,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:02:27,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:32,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:32,656 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:37,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:37,657 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:39,268 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:02:41,879 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:02:43,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:47,658 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:48,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:52,659 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:02:53,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:02:56,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:02:58,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:02,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:03,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:07,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:07,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:03:08,002 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:03:08,002 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:03:08,004 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:03:08,464 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:03:09,004 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:09,269 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:03:09,464 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:03:11,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:03:14,269 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:17,662 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:19,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:22,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:24,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:26,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:03:29,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:32,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:34,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:37,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:39,269 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:03:40,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:41,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:03:45,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:47,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:50,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:52,667 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:03:55,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:03:56,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:04:00,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:02,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:05,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:07,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:09,270 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:04:10,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:04:10,478 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:04:10,479 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:04:10,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:10,481 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:04:10,501 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:04:11,502 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:04:11,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:04:15,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:17,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:20,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:22,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:25,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:26,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:04:30,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:32,673 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:35,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:37,674 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:39,271 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:04:41,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:41,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:04:46,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:47,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:51,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:52,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:04:56,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:04:56,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:05:01,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:02,678 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:06,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:07,679 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:09,272 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:05:11,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:05:12,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:13,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:05:13,015 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:05:13,016 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:05:13,017 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:05:13,540 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:05:15,542 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:05:17,680 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:18,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:22,681 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:23,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:26,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:05:28,056 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:32,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:33,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:37,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:38,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:39,273 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:05:41,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:05:43,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:47,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:48,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:52,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:05:53,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:05:56,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:05:58,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:02,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:03,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:07,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:08,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:09,274 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:06:11,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:06:14,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:15,654 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:06:15,656 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:06:15,657 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:06:15,658 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:06:16,580 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:06:17,580 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:06:17,727 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:19,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:22,728 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:24,699 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:26,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:06:29,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:32,729 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:34,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:37,730 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:39,275 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:06:40,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:41,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:06:45,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:47,731 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:50,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:52,732 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:06:55,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:06:56,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:07:00,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:02,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:05,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:07,734 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:09,276 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:07:11,276 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:11,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:07:16,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:17,735 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:18,074 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:07:18,076 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:07:18,076 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:07:18,078 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:07:18,616 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:07:19,617 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:07:22,078 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:22,736 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:26,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:07:27,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:32,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:32,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:37,080 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:37,739 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:39,276 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:07:41,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:07:42,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:47,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:47,740 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:52,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:07:52,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:07:56,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:07:57,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:02,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:02,788 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:07,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:07,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:09,277 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:08:11,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:08:13,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:17,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:18,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:20,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:08:20,351 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:08:20,351 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:08:20,353 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:08:20,653 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:08:21,654 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:08:22,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:23,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:26,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:08:28,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:32,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:33,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:37,793 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:38,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:39,278 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:08:41,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:08:44,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:47,848 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:49,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:52,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:08:54,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:08:56,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:08:59,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:02,850 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:04,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:07,851 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:09,279 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:09:10,279 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:11,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:09:15,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:17,852 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:20,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:22,733 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:09:22,735 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:09:22,735 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:09:22,737 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:09:22,853 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:23,690 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:09:23,691 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:09:25,738 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:26,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:09:30,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:32,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:35,750 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:37,855 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:39,280 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:09:41,280 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:41,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:09:46,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:47,857 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:51,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:52,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:09:56,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:09:56,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:10:01,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:02,859 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:06,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:07,860 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:09,281 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:10:11,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:10:12,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:17,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:17,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:22,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:22,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:25,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:10:25,396 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:10:25,396 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:10:25,397 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:10:25,727 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:10:26,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:10:27,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:28,728 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:10:31,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:32,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:37,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:37,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:39,281 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:10:41,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:10:43,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:47,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:48,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:52,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:10:53,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:10:56,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:10:58,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:02,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:03,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:07,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:08,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:09,282 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:11:11,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:11:14,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:17,870 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:19,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:22,952 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:24,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:26,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:11:27,646 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:11:27,648 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:11:27,648 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:11:27,654 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:11:27,763 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:11:28,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:11:30,157 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:30,764 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:11:32,953 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:35,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:37,954 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:39,283 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:11:40,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:41,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:11:45,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:47,956 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:50,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:52,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:11:55,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:11:56,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:12:00,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:02,958 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:05,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:07,959 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:09,284 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:12:11,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:11,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:12:16,285 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:17,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:21,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:22,961 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:26,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:26,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:12:30,875 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:12:30,877 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:12:30,878 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:12:30,879 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:12:31,798 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:12:31,919 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:32,799 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:12:32,962 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:36,920 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:37,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:39,287 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:12:41,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:12:42,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:46,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:47,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:51,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:12:52,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:12:56,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:12:57,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:01,967 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:02,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:06,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:07,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:09,286 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:13:11,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:13:13,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:16,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:18,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:21,970 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:23,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:26,963 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:13:28,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:31,971 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:33,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:13:33,289 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:13:33,290 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:13:33,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:33,292 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:13:33,832 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:13:34,833 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:13:36,972 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:38,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:39,287 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:13:41,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:13:44,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:46,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:49,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:52,047 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:13:54,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:13:56,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:13:59,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:02,048 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:04,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:07,049 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:09,289 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:14:10,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:11,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:14:15,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:17,050 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:20,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:22,051 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:25,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:26,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:14:30,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:32,052 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:35,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:35,904 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:14:35,906 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:14:35,906 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:14:35,907 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:14:36,868 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:14:37,053 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:38,870 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:14:39,288 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:14:41,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:41,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:14:46,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:47,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:51,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:52,055 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:14:56,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:14:56,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:15:01,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:02,057 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:06,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:07,058 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:09,290 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:15:11,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:15:12,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:17,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:17,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:22,060 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:22,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:26,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:15:27,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:32,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:32,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:37,062 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:37,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:38,094 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:15:38,097 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:15:38,097 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:15:38,098 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:15:38,904 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:15:39,290 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:15:40,906 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:15:41,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:15:43,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:47,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:48,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:52,064 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:15:53,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:15:56,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:15:58,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:02,066 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:03,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:07,067 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:08,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:09,291 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:16:11,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:16:14,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:17,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:19,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:22,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:24,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:27,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:16:29,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:32,142 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:34,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:37,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:39,292 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:16:40,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:40,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:16:40,395 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:16:40,395 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:16:40,396 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:16:40,940 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:16:42,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:16:42,941 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:16:45,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:47,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:50,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:52,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:16:55,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:16:57,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:17:00,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:02,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:05,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:07,203 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:09,293 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:17:11,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:12,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:17:16,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:17,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:21,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:22,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:26,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:27,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:17:31,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:32,207 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:36,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:37,259 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:39,294 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:17:42,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:17:42,295 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:43,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:17:43,121 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:17:43,121 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:17:43,122 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:17:43,977 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:17:44,978 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:17:47,260 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:48,163 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:52,261 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:17:53,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:17:57,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:17:58,164 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:02,263 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:03,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:07,264 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:08,165 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:09,295 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:18:12,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:18:13,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:17,265 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:18,296 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:22,266 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:23,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:27,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:18:28,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:32,267 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:33,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:37,268 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:38,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:39,296 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:18:42,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:18:44,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:45,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:18:45,286 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:18:45,286 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:18:45,287 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:18:46,013 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:18:47,014 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:18:47,270 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:49,327 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:52,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:18:54,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:18:57,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:18:59,328 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:02,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:04,329 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:07,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:09,297 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:19:10,297 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:12,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:19:15,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:17,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:20,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:22,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:25,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:27,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:19:30,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:32,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:35,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:37,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:39,297 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:19:41,298 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:42,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:19:46,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:47,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:47,960 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:19:47,961 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:19:47,962 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:19:47,963 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:19:48,049 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:19:49,049 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:19:51,964 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:52,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:19:56,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:19:57,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:20:01,965 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:02,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:06,966 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:07,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:09,300 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:20:12,084 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:20:12,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:17,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:17,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:22,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:22,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:27,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:20:27,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:32,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:32,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:37,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:37,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:39,299 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:20:42,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:20:43,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:47,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:48,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:50,595 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:20:50,597 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:20:50,598 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:20:50,600 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:20:51,086 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:20:52,477 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:20:53,087 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:20:53,601 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:20:57,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:20:58,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:02,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:03,602 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:07,479 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:08,603 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:09,300 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:21:12,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:21:14,301 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:17,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:19,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:22,481 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:24,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:27,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:21:29,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:32,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:34,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:37,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:39,301 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:21:40,302 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:42,085 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:21:45,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:47,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:50,303 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:52,485 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:21:52,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:21:52,939 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:21:52,940 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:21:52,942 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:21:53,123 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:21:55,124 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:21:55,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:21:57,221 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:22:00,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:02,503 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:02,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:05,131 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:22:06,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:08,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:09,302 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:22:11,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:12,135 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:22:12,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:22:13,136 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:22:16,506 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:18,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:21,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:23,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:25,144 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:22:27,145 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:22:27,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:22:27,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:32,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:32,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:37,425 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:37,494 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:39,303 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:22:42,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:22:43,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:47,426 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:48,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:52,427 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:22:53,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:22:57,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:22:58,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:02,429 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:03,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:07,430 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:08,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:09,304 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:23:12,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:23:14,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:17,431 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:19,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:22,432 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:24,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:27,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:23:28,628 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:23:28,630 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:23:28,631 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:23:28,632 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:23:29,183 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:23:29,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:31,184 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:23:32,433 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:34,671 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:37,434 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:39,304 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:23:40,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:42,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:23:45,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:47,435 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:50,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:52,436 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:23:55,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:23:57,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:24:00,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:02,437 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:05,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:07,439 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:09,305 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:24:11,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:12,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:24:16,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:17,440 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:21,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:22,441 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:26,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:27,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:24:31,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:31,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:24:31,780 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:24:31,780 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:24:31,782 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:24:32,218 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:24:32,442 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:33,219 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:24:36,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:37,443 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:39,306 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:24:42,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:24:42,307 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:47,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:47,444 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:52,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:24:52,445 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:24:57,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:24:57,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:02,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:02,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:07,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:07,447 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:09,307 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:25:12,271 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:25:13,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:17,448 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:18,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:22,449 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:23,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:27,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:25:28,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:32,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:33,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:34,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:25:34,542 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:25:34,543 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:25:34,544 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:25:35,254 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:25:37,255 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:25:37,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:38,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:39,308 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:25:42,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:25:44,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:47,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:49,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:52,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:25:54,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:25:57,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:25:59,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:02,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:04,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:07,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:09,309 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:26:10,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:12,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:26:15,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:17,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:20,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:22,478 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:25,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:27,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:26:30,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:32,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:35,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:36,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:26:36,844 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:26:36,845 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:26:36,846 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:26:37,289 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:26:37,480 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:39,290 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:26:39,310 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:26:41,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:42,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:26:46,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:47,482 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:51,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:52,483 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:26:56,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:26:57,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:27:01,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:02,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:06,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:07,484 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:09,311 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:27:12,273 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:27:12,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:17,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:17,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:22,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:22,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:27,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:27:27,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:32,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:32,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:37,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:37,489 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:39,313 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:27:39,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:27:39,404 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:27:39,404 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:27:39,406 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:27:40,339 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:27:41,340 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:27:42,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:27:42,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:47,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:47,490 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:52,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:27:52,491 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:27:57,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:27:57,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:02,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:02,492 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:07,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:07,493 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:09,313 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:28:12,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:28:13,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:17,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:18,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:22,515 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:23,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:27,275 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:28:28,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:32,529 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:33,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:37,570 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:38,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:39,315 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:28:41,644 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:28:41,647 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:28:41,647 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:28:41,648 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:28:42,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:28:42,380 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:28:43,381 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:28:43,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:47,571 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:48,649 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:52,572 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:28:53,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:28:57,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:28:58,650 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:02,573 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:03,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:07,574 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:08,651 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:09,316 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:29:12,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:29:14,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:17,575 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:19,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:22,576 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:24,318 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:27,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:29:29,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:32,578 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:34,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:37,579 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:39,317 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:29:40,317 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:42,274 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:29:44,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:29:44,113 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:29:44,114 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:29:44,115 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:29:44,430 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:29:45,431 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:29:46,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:47,581 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:51,155 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:52,582 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:29:56,156 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:29:57,277 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:30:01,158 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:02,664 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:06,159 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:07,665 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:09,318 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:30:11,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:12,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:30:16,319 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:17,666 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:21,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:22,668 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:26,320 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:27,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:30:31,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:32,669 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:36,321 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:37,670 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:39,340 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:30:41,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:42,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:30:46,341 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:46,754 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:30:46,756 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:30:46,756 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:30:46,758 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:30:47,469 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:30:47,700 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:49,470 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:30:51,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:52,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:30:56,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:30:57,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:31:01,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:02,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:06,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:07,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:09,341 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:31:12,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:31:12,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:17,342 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:17,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:22,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:22,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:27,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:31:27,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:32,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:32,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:37,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:37,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:39,342 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:31:42,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:31:43,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:47,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:48,343 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:49,070 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:31:49,072 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:31:49,072 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:31:49,074 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:31:49,504 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:31:51,506 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:31:52,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:31:54,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:31:57,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:31:59,075 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:02,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:04,076 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:07,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:09,077 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:09,343 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:32:12,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:32:14,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:17,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:19,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:22,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:24,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:27,278 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:32:29,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:32,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:34,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:37,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:39,344 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:32:40,344 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:42,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:32:45,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:47,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:50,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:51,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:32:51,864 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:32:51,864 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:32:51,866 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:32:52,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:32:52,541 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:32:52,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:32:53,542 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:32:55,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:32:57,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:33:00,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:02,743 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:05,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:07,744 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:09,345 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:33:11,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:12,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:33:16,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:17,745 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:21,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:22,746 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:26,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:27,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:33:31,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:32,772 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:36,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:37,773 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:39,346 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:33:42,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:33:42,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:47,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:47,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:52,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:33:52,775 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:33:54,912 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:33:54,914 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:33:54,915 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:33:54,916 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:33:55,574 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:33:57,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:33:57,576 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:33:57,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:02,777 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:02,917 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:07,778 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:07,918 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:09,347 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:34:12,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:34:13,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:17,779 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:18,348 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:22,780 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:23,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:27,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:34:28,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:32,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:33,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:37,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:38,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:39,348 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:34:42,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:34:44,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:47,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:49,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:52,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:34:54,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:34:57,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:34:57,660 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:34:57,664 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:34:57,664 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:34:57,665 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:34:58,609 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:34:59,609 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:34:59,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:02,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:04,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:07,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:09,348 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:35:10,349 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:12,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:35:15,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:17,833 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:20,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:22,834 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:25,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:27,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:35:30,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:32,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:35,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:37,837 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:39,349 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:35:41,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:42,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:35:46,350 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:47,838 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:51,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:52,840 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:35:56,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:35:57,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:36:00,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:36:00,467 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:36:00,467 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:36:00,468 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:36:00,642 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:36:01,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:01,643 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:36:02,926 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:06,507 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:07,927 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:09,350 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:36:12,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:36:12,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:17,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:17,929 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:22,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:22,930 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:27,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:36:27,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:32,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:32,931 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:37,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:37,932 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:39,351 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:36:42,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:36:43,352 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:47,934 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:48,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:52,935 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:36:53,353 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:36:57,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:36:58,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:02,937 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:03,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:03,522 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:37:03,524 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:37:03,524 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:37:03,530 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:37:03,689 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:37:05,690 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:37:07,938 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:08,531 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:09,354 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:37:12,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:37:14,355 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:17,939 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:19,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:22,940 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:24,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:27,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:37:29,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:32,941 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:34,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:37,942 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:39,355 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:37:40,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:42,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:37:45,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:47,943 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:50,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:52,944 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:37:55,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:37:57,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:38:00,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:02,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:05,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:05,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:38:05,910 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:38:05,910 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:38:05,912 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:38:06,743 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:38:07,743 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:38:07,947 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:09,356 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:38:11,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:12,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:38:16,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:17,986 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:21,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:22,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:26,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:27,283 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:38:31,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:32,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:36,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:37,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:39,357 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:38:42,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:38:42,357 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:47,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:47,991 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:52,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:38:52,992 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:38:57,284 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:38:57,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:02,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:02,993 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:07,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:07,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:08,446 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:39:08,447 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:39:08,448 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:39:08,450 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:39:08,814 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:39:09,358 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:39:09,815 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:39:12,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:39:13,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:17,995 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:18,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:22,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:23,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:27,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:39:28,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:32,998 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:33,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:37,999 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:38,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:39,359 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:39:42,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:39:44,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:48,000 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:49,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:53,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:39:54,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:39:57,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:39:59,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:03,002 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:04,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:08,003 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:09,360 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:40:10,360 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:11,249 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:40:11,251 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:40:11,251 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:40:11,252 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:40:11,853 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:40:11,853 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:40:12,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:40:16,253 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:18,005 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:21,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:23,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:26,254 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:27,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:40:31,255 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:33,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:36,258 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:38,010 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:39,360 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:40:41,361 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:42,286 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:40:46,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:48,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:51,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:53,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:40:56,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:40:57,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:41:01,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:03,015 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:06,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:08,016 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:09,361 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:41:12,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:41:12,362 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:13,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:41:13,635 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:41:13,636 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:41:13,637 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:41:13,889 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:41:15,890 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:41:17,675 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:18,017 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:22,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:23,018 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:27,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:41:27,676 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:32,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:33,119 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:37,677 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:38,120 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:39,362 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:41:42,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:41:43,363 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:48,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:48,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:53,123 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:41:53,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:41:57,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:41:58,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:03,124 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:03,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:08,281 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:08,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:09,363 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:42:12,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:42:14,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:16,106 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:42:16,108 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:42:16,108 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:42:16,134 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:42:16,927 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:42:17,928 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:42:18,282 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:20,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:23,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:25,151 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:27,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:42:30,152 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:32,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:35,153 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:37,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:39,367 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:42:40,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:42,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:42:45,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:47,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:50,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:52,316 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:42:55,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:42:57,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:43:00,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:02,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:05,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:07,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:09,365 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:43:11,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:12,287 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:43:16,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:17,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:18,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:43:18,629 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:43:18,629 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:43:18,631 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:43:18,964 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:43:19,964 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:43:21,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:22,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:26,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:27,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:43:31,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:32,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:36,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:37,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:39,366 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:43:42,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:43:42,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:47,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:47,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:52,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:43:52,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:43:57,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:43:57,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:02,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:02,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:07,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:07,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:09,368 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:44:12,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:44:13,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:17,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:18,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:20,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:44:20,635 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:44:20,635 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:44:20,637 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:44:20,999 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:44:22,000 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:44:22,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:23,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:27,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:44:28,638 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:32,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:33,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:37,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:38,639 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:39,368 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:44:42,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:44:44,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:47,466 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:49,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:52,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:44:54,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:44:57,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:44:59,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:02,534 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:04,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:07,535 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:09,369 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:45:10,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:12,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:45:15,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:17,536 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:20,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:22,537 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:23,176 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:45:23,179 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:45:23,179 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:45:23,180 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:45:24,047 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:45:24,048 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:45:26,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:27,288 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:45:31,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:32,539 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:36,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:37,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:39,369 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:45:41,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:42,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:45:46,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:47,541 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:51,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:52,542 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:45:56,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:45:57,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:46:01,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:02,544 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:06,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:07,545 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:09,370 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:46:12,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:46:12,371 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:17,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:17,546 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:22,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:22,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:26,001 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:46:26,003 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:46:26,004 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:46:26,005 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:46:26,084 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:46:27,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:46:28,006 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:28,086 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:46:32,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:33,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:37,549 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:38,007 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:39,371 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:46:42,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:46:43,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:47,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:48,372 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:52,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:46:53,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:46:57,289 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:46:58,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:02,606 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:03,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:07,607 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:08,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:09,372 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:47:12,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:47:14,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:17,608 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:19,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:22,609 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:24,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:27,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:47:28,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:47:28,308 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:47:28,308 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:47:28,310 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:47:29,121 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:47:30,122 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:47:30,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:32,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:35,351 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:37,614 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:39,372 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:47:40,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:42,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:47:45,373 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:47,615 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:50,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:52,616 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:47:55,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:47:57,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:48:00,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:02,617 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:05,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:07,618 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:09,373 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:48:11,374 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:12,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:48:16,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:17,619 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:21,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:22,620 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:26,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:27,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:48:31,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:48:31,138 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:48:31,139 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:48:31,140 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:48:31,159 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:48:32,159 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:48:32,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:32,621 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:37,179 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:37,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:39,374 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:48:42,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:48:42,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:47,375 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:47,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:52,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:48:52,624 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:48:57,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:48:57,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:02,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:02,626 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:07,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:07,627 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:09,375 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:49:12,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:49:13,376 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:17,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:18,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:22,629 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:23,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:27,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:49:28,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:32,630 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:33,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:33,849 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:49:33,851 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:49:33,851 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:49:33,853 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:49:34,196 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:49:37,199 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:49:37,631 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:38,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:39,376 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:49:42,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:49:44,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:47,632 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:49,377 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:52,633 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:49:54,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:49:57,290 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:49:59,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:02,634 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:04,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:07,635 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:09,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:09,391 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:50:12,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:50:14,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:17,636 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:19,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:22,637 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:24,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:27,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:50:29,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:32,717 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:34,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:36,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:50:36,391 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:50:36,391 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:50:36,393 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:50:37,234 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:50:37,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:39,235 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:50:39,378 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:50:40,378 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:42,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:50:45,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:47,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:50,379 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:52,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:50:55,380 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:50:57,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:51:00,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:02,722 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:05,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:07,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:09,381 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:51:10,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:12,291 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:51:15,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:17,817 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:20,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:22,818 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:25,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:27,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:51:30,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:32,820 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:35,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:37,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:38,973 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:51:38,975 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:51:38,976 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:51:38,977 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:51:39,269 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:51:39,381 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:51:41,271 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:51:41,381 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:42,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:51:46,382 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:47,822 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:51,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:52,823 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:51:56,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:51:57,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:52:01,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:02,825 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:06,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:07,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:09,382 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:52:12,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:52:12,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:17,383 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:17,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:22,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:22,828 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:27,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:52:27,384 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:32,385 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:32,854 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:37,386 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:37,856 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:39,383 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:52:41,552 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:52:41,554 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:52:41,555 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:52:41,556 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:52:42,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:52:42,306 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:52:42,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:43,307 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:52:47,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:47,858 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:52,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:52:52,859 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:52:57,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:52:57,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:02,559 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:02,861 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:07,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:07,862 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:09,387 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:53:12,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:53:13,387 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:17,864 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:18,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:22,865 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:23,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:27,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:53:28,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:32,866 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:33,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:37,867 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:38,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:39,387 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:53:42,292 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:53:44,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:44,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:53:44,407 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:53:44,408 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:53:44,409 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:53:45,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:53:45,347 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:53:47,348 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:53:47,868 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:50,211 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:52,869 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:53:55,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:53:57,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:54:00,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:02,898 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:05,213 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:07,899 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:09,388 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:54:10,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:12,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:54:15,389 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:17,900 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:20,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:22,901 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:25,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:27,293 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:54:30,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:32,984 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:35,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:37,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:39,389 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:54:41,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:42,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:54:46,390 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:47,987 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:48,122 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:54:48,125 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:54:48,125 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:54:48,127 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:54:48,421 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:54:49,422 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:54:52,127 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:52,988 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:54:57,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:54:57,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:55:02,128 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:02,989 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:07,129 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:07,990 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:09,391 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:55:12,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:55:12,391 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:17,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:18,008 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:22,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:23,009 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:27,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:55:27,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:32,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:33,011 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:37,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:38,012 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:39,392 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:55:42,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:55:43,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:48,013 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:48,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:51,177 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:55:51,179 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:55:51,179 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:55:51,181 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:55:51,456 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:55:53,014 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:55:53,457 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:55:54,181 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:55:57,294 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:55:59,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:03,033 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:04,182 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:08,034 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:09,183 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:09,392 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:56:12,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:56:14,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:18,035 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:19,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:23,036 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:24,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:27,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:56:29,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:33,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:34,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:38,038 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:39,393 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:56:40,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:42,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:56:45,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:48,039 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:50,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:53,040 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:56:53,821 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:56:53,823 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:56:53,824 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:56:53,825 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:56:54,490 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:56:55,491 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:56:55,826 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:56:57,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:57:00,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:03,041 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:05,827 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:08,042 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:09,394 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:57:11,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:12,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:57:16,395 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:18,043 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:21,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:23,044 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:26,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:27,392 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:57:31,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:33,131 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:36,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:38,132 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:39,395 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:57:42,396 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:42,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:57:47,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:48,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:52,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:53,134 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:57:56,540 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:57:56,545 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:57:56,545 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:57:56,547 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:57:57,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:57:57,526 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:57:57,547 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:57:59,527 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:58:02,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:03,136 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:07,548 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:08,137 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:09,396 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:58:12,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:58:13,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:18,139 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:18,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:23,140 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:23,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:27,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:58:28,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:33,141 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:33,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:38,142 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:38,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:39,397 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:58:42,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:58:44,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:48,143 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:49,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:53,197 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:58:54,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:58:57,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:58:58,756 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 10:58:58,758 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 10:58:58,758 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 10:58:58,759 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 10:58:59,561 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 10:58:59,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:01,562 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 10:59:03,199 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:04,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:08,200 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:09,398 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:59:10,398 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:12,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:59:15,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:18,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:20,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:23,202 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:25,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:27,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:59:30,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:33,204 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:35,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:38,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:39,399 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 10:59:41,399 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:42,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 10:59:46,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:48,206 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:51,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:53,208 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 10:59:56,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 10:59:57,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:00:01,037 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:00:01,039 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:00:01,039 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:00:01,041 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:00:01,598 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:00:02,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:03,209 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:03,600 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:00:07,079 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:08,210 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:09,400 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:00:12,400 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:12,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:00:17,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:18,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:22,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:23,212 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:27,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:27,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:00:32,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:33,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:37,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:38,215 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:39,401 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:00:42,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:00:43,401 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:48,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:48,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:53,310 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:00:53,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:00:57,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:00:58,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:03,311 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:03,354 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:01:03,356 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:01:03,356 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:01:03,358 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:01:03,634 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:01:04,358 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:05,635 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:01:08,312 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:09,359 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:09,401 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:01:12,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:01:14,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:18,313 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:19,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:23,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:24,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:27,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:01:29,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:33,335 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:34,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:38,364 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:39,402 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:01:40,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:42,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:01:45,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:48,365 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:50,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:53,366 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:01:55,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:01:57,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:02:00,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:03,367 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:05,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:05,528 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:02:05,530 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:02:05,530 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:02:05,532 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:02:05,704 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:02:07,705 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:02:08,368 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:09,403 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:02:11,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:12,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:02:16,404 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:18,369 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:21,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:23,370 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:26,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:27,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:02:31,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:33,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:36,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:38,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:39,404 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:02:42,405 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:42,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:02:47,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:47,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:52,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:52,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:02:57,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:02:57,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:03:02,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:02,455 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:07,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:07,456 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:08,205 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:03:08,208 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:03:08,208 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:03:08,209 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:03:08,745 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:03:09,405 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:03:09,745 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:03:12,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:03:13,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:17,457 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:18,406 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:22,458 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:23,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:27,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:03:28,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:32,459 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:33,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:37,461 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:38,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:39,406 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:03:42,450 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:03:44,407 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:47,462 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:49,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:52,463 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:03:54,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:03:57,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:03:59,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:02,464 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:04,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:07,465 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:09,408 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:04:10,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:10,903 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:04:10,905 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:04:10,906 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:04:10,907 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:04:11,779 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:04:12,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:04:13,780 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:04:15,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:17,466 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:20,908 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:22,467 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:25,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:27,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:04:30,909 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:32,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:35,910 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:37,469 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:39,408 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:04:41,408 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:42,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:04:46,409 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:47,471 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:51,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:52,472 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:04:56,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:04:57,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:05:01,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:02,473 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:06,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:07,474 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:09,409 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:05:12,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:12,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:05:13,299 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:05:13,301 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:05:13,301 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:05:13,302 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:05:13,814 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:05:15,815 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:05:17,475 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:18,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:22,476 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:23,345 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:27,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:05:28,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:32,550 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:33,346 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:37,551 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:38,347 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:39,410 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:05:42,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:05:43,410 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:47,553 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:48,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:52,556 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:05:53,411 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:05:57,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:05:58,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:02,557 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:03,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:07,558 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:08,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:09,411 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:06:12,451 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:06:14,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:15,760 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:06:15,761 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:06:15,762 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:06:15,763 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:06:15,847 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:06:17,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:17,848 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:06:19,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:22,560 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:24,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:27,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:06:29,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:32,622 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:34,805 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:37,623 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:39,411 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:06:40,412 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:42,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:06:45,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:47,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:50,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:52,661 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:06:55,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:06:57,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:07:00,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:02,663 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:05,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:07,701 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:09,412 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:07:11,413 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:12,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:07:16,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:17,702 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:18,397 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:07:18,399 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:07:18,399 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:07:18,401 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:07:18,882 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:07:19,882 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:07:22,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:22,703 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:27,402 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:27,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:07:32,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:32,704 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:37,403 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:37,705 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:39,413 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:07:42,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:42,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:07:47,414 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:47,707 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:52,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:52,708 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:07:57,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:07:57,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:08:02,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:02,709 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:07,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:07,710 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:09,414 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:08:12,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:08:13,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:17,711 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:18,415 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:21,112 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:08:21,114 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:08:21,114 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:08:21,116 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:08:21,917 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:08:22,712 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:23,918 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:08:24,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:27,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:08:29,117 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:32,713 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:34,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:37,714 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:39,118 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:39,415 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:08:42,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:08:44,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:47,715 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:49,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:52,716 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:08:54,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:08:57,453 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:08:59,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:02,718 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:04,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:07,719 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:09,416 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:09:10,416 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:12,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:09:15,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:17,720 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:20,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:22,721 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:23,388 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:09:23,390 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:09:23,390 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:09:23,392 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:09:23,952 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:09:25,953 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:09:26,393 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:27,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:09:31,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:32,723 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:36,394 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:37,724 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:39,417 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:09:41,417 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:42,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:09:46,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:47,725 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:51,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:52,781 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:09:56,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:09:57,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:10:01,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:02,782 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:06,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:07,783 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:09,418 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:10:12,418 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:12,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:10:17,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:17,784 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:22,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:22,785 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:25,836 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:10:25,838 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:10:25,838 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:10:25,840 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:10:25,988 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:10:27,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:10:27,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:27,989 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:10:32,786 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:32,841 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:37,787 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:37,842 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:39,418 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:10:42,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:10:43,419 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:47,789 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:48,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:52,790 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:10:53,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:10:57,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:10:58,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:02,791 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:03,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:07,792 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:08,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:09,419 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:11:12,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:11:14,420 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:17,794 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:19,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:22,795 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:24,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:27,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:11:28,272 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:11:28,273 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:11:28,274 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:11:28,275 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:11:29,023 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:11:30,024 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:11:30,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:32,796 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:35,315 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:37,797 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:39,420 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:11:40,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:42,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:11:45,421 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:47,799 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:50,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:52,800 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:11:55,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:11:57,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:12:00,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:02,801 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:05,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:07,802 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:09,421 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:12:11,422 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:12,486 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:12:16,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:17,803 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:21,423 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:22,804 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:26,424 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:27,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:12:30,957 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 11:12:30,959 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:12:30,959 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:12:30,961 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:12:31,058 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:12:32,059 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:12:32,233 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:32,563 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:37,234 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:38,452 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:39,428 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:12:42,065 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:12:42,487 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:12:42,968 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:47,526 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:47,969 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:50,070 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:12:52,985 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:12:53,300 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:12:57,604 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:12:58,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:13:03,356 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:13:03,592 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: keepalive +2024-11-01 11:13:03,994 DEBUG SenderThread:31085 [sender.py:send():382] send: exit +2024-11-01 11:13:03,995 INFO SenderThread:31085 [sender.py:send_exit():589] handling exit code: 0 +2024-11-01 11:13:03,995 INFO SenderThread:31085 [sender.py:send_exit():591] handling runtime: 44306 +2024-11-01 11:13:03,996 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:13:03,996 INFO SenderThread:31085 [sender.py:send_exit():597] send defer +2024-11-01 11:13:03,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:03,996 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 0 +2024-11-01 11:13:03,996 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:03,997 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-11-01 11:13:03,997 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 1 +2024-11-01 11:13:03,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:03,997 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 1 +2024-11-01 11:13:03,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:03,997 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-11-01 11:13:03,997 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 2 +2024-11-01 11:13:03,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:03,997 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 2 +2024-11-01 11:13:03,997 INFO HandlerThread:31085 [system_monitor.py:finish():203] Stopping system monitor +2024-11-01 11:13:03,997 INFO HandlerThread:31085 [interfaces.py:finish():202] Joined cpu monitor +2024-11-01 11:13:03,997 DEBUG SystemMonitor:31085 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-11-01 11:13:03,998 INFO HandlerThread:31085 [interfaces.py:finish():202] Joined disk monitor +2024-11-01 11:13:03,998 DEBUG SystemMonitor:31085 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-11-01 11:13:04,079 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:13:04,079 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:13:04,184 INFO HandlerThread:31085 [interfaces.py:finish():202] Joined gpu monitor +2024-11-01 11:13:04,184 INFO HandlerThread:31085 [interfaces.py:finish():202] Joined memory monitor +2024-11-01 11:13:04,184 INFO HandlerThread:31085 [interfaces.py:finish():202] Joined network monitor +2024-11-01 11:13:04,185 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:04,185 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-11-01 11:13:04,185 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 3 +2024-11-01 11:13:04,185 DEBUG SenderThread:31085 [sender.py:send():382] send: stats +2024-11-01 11:13:04,185 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:04,185 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 3 +2024-11-01 11:13:04,187 DEBUG SenderThread:31085 [sender.py:send():382] send: history +2024-11-01 11:13:04,187 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: summary_record +2024-11-01 11:13:04,188 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:13:04,188 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:04,188 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-11-01 11:13:04,188 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 4 +2024-11-01 11:13:04,188 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:04,188 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 4 +2024-11-01 11:13:04,189 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:04,189 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-11-01 11:13:04,189 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 5 +2024-11-01 11:13:04,189 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:04,189 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 5 +2024-11-01 11:13:04,190 DEBUG SenderThread:31085 [sender.py:send():382] send: summary +2024-11-01 11:13:04,190 INFO SenderThread:31085 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 11:13:04,191 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:04,191 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-11-01 11:13:04,191 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 6 +2024-11-01 11:13:04,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:04,191 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 6 +2024-11-01 11:13:04,191 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:04,191 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-11-01 11:13:04,191 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 7 +2024-11-01 11:13:04,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 11:13:04,191 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:04,191 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 7 +2024-11-01 11:13:04,192 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:04,192 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-11-01 11:13:04,994 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 11:13:05,080 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:13:06,080 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:13:07,213 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 8 +2024-11-01 11:13:07,214 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 11:13:07,214 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:07,214 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 8 +2024-11-01 11:13:07,214 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:07,214 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-11-01 11:13:07,214 INFO SenderThread:31085 [job_builder.py:build():296] Attempting to build job artifact +2024-11-01 11:13:07,230 INFO SenderThread:31085 [job_builder.py:_get_source_type():426] is repo sourced job +2024-11-01 11:13:07,884 INFO SenderThread:31085 [job_builder.py:build():402] adding wandb-job metadata file +2024-11-01 11:13:07,944 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 9 +2024-11-01 11:13:07,945 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:07,945 DEBUG SenderThread:31085 [sender.py:send():382] send: artifact +2024-11-01 11:13:07,945 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 9 +2024-11-01 11:13:07,996 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 11:13:08,083 INFO Thread-12 :31085 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:13:08,846 INFO SenderThread:31085 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTMwMzIzMjA4MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQ4MzQ0MjQxNg==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTMwMzIzMjA4MQ==', 'versionIndex': 2}}} +2024-11-01 11:13:08,847 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:08,847 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-11-01 11:13:08,847 INFO SenderThread:31085 [dir_watcher.py:finish():358] shutting down directory watcher +2024-11-01 11:13:09,084 INFO SenderThread:31085 [dir_watcher.py:finish():388] scan: /project/wandb/run-20241031_225437-1g18vgup/files +2024-11-01 11:13:09,084 INFO SenderThread:31085 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_225437-1g18vgup/files/requirements.txt requirements.txt +2024-11-01 11:13:09,085 INFO SenderThread:31085 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_225437-1g18vgup/files/config.yaml config.yaml +2024-11-01 11:13:09,086 INFO SenderThread:31085 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-metadata.json wandb-metadata.json +2024-11-01 11:13:09,087 INFO SenderThread:31085 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json wandb-summary.json +2024-11-01 11:13:09,087 INFO SenderThread:31085 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_225437-1g18vgup/files/output.log output.log +2024-11-01 11:13:09,090 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 10 +2024-11-01 11:13:09,090 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 11:13:09,090 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:09,092 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 10 +2024-11-01 11:13:09,092 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:09,092 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-11-01 11:13:09,092 INFO SenderThread:31085 [file_pusher.py:finish():172] shutting down file pusher +2024-11-01 11:13:09,550 INFO wandb-upload_2:31085 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_225437-1g18vgup/files/wandb-summary.json +2024-11-01 11:13:09,550 INFO wandb-upload_0:31085 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_225437-1g18vgup/files/requirements.txt +2024-11-01 11:13:09,568 INFO wandb-upload_1:31085 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_225437-1g18vgup/files/config.yaml +2024-11-01 11:13:09,912 INFO wandb-upload_3:31085 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_225437-1g18vgup/files/output.log +2024-11-01 11:13:09,997 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 11:13:09,997 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 11:13:10,113 INFO Thread-11 (_thread_body):31085 [sender.py:transition_state():617] send defer: 11 +2024-11-01 11:13:10,113 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:10,113 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 11 +2024-11-01 11:13:10,114 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:10,114 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-11-01 11:13:10,114 INFO SenderThread:31085 [file_pusher.py:join():178] waiting for file pusher +2024-11-01 11:13:10,114 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 12 +2024-11-01 11:13:10,114 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:10,114 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 12 +2024-11-01 11:13:10,114 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:10,114 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-11-01 11:13:10,114 INFO SenderThread:31085 [file_stream.py:finish():595] file stream finish called +2024-11-01 11:13:10,303 INFO SenderThread:31085 [file_stream.py:finish():599] file stream finish is done +2024-11-01 11:13:10,303 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 13 +2024-11-01 11:13:10,304 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:10,304 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 13 +2024-11-01 11:13:10,304 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:10,304 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-11-01 11:13:10,304 INFO SenderThread:31085 [sender.py:transition_state():617] send defer: 14 +2024-11-01 11:13:10,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: defer +2024-11-01 11:13:10,305 DEBUG SenderThread:31085 [sender.py:send():382] send: final +2024-11-01 11:13:10,305 INFO HandlerThread:31085 [handler.py:handle_request_defer():172] handle defer: 14 +2024-11-01 11:13:10,305 DEBUG SenderThread:31085 [sender.py:send():382] send: footer +2024-11-01 11:13:10,305 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: defer +2024-11-01 11:13:10,305 INFO SenderThread:31085 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-11-01 11:13:10,305 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 11:13:10,306 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 11:13:10,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 11:13:10,306 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: server_info +2024-11-01 11:13:10,306 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 11:13:10,307 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: server_info +2024-11-01 11:13:10,308 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: get_summary +2024-11-01 11:13:10,309 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: sampled_history +2024-11-01 11:13:10,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 11:13:10,314 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: job_info +2024-11-01 11:13:10,492 DEBUG SenderThread:31085 [sender.py:send_request():409] send_request: job_info +2024-11-01 11:13:10,492 INFO MainThread:31085 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-11-01 11:13:10,494 INFO MainThread:31085 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-11-01 11:13:10,494 INFO MainThread:31085 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-11-01 11:13:10,495 DEBUG HandlerThread:31085 [handler.py:handle_request():146] handle_request: shutdown +2024-11-01 11:13:10,495 INFO HandlerThread:31085 [handler.py:finish():869] shutting down handler +2024-11-01 11:13:11,315 INFO WriterThread:31085 [datastore.py:close():296] close: /project/wandb/run-20241031_225437-1g18vgup/run-1g18vgup.wandb +2024-11-01 11:13:11,492 INFO SenderThread:31085 [sender.py:finish():1572] shutting down sender +2024-11-01 11:13:11,492 INFO SenderThread:31085 [file_pusher.py:finish():172] shutting down file pusher +2024-11-01 11:13:11,493 INFO SenderThread:31085 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20241031_225437-1g18vgup/logs/debug.log b/wandb/run-20241031_225437-1g18vgup/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b76dc2ac2bcbd156f1e2290fd27ffc35c1a3bc2d --- /dev/null +++ b/wandb/run-20241031_225437-1g18vgup/logs/debug.log @@ -0,0 +1,30 @@ +2024-10-31 22:54:37,315 INFO MainThread:30984 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_setup.py:_flush():76] Configure stats pid to 30984 +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'CPT en-ja'} +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20241031_225437-1g18vgup/logs/debug.log +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20241031_225437-1g18vgup/logs/debug-internal.log +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_init.py:init():566] calling init triggers +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['183190553', '/project/trans/datasets/train/documents_text_document'], 'valid_data_path': ['61085876', '/project/trans/datasets/valid/documents_text_document'], 'test_data_path': ['61085876', '/project/trans/datasets/valid/documents_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-22:54:20', 'wandb_project': 'trans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', 'save': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', 'base_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 20, 'save_interval': 50, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 0.00015, 'lr_decay_style': 'cosine', 'lr_decay_iters': 698, 'lr_warmup_iters': 70, 'min_lr': 1.5e-05, 'train_iters': 698, 'train_samples': None, 'global_batch_size': 256, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 3, 'hf_repo_id': 'koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 99584, 'gradient_accumulation_steps': 256} +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_init.py:init():616] starting backend +2024-10-31 22:54:37,316 INFO MainThread:30984 [wandb_init.py:init():620] setting up manager +2024-10-31 22:54:37,321 INFO MainThread:30984 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-31 22:54:37,322 INFO MainThread:30984 [wandb_init.py:init():628] backend started and connected +2024-10-31 22:54:37,327 INFO MainThread:30984 [wandb_init.py:init():720] updated telemetry +2024-10-31 22:54:37,338 INFO MainThread:30984 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-10-31 22:54:37,789 INFO MainThread:30984 [wandb_run.py:_on_init():2262] communicating current version +2024-10-31 22:54:37,817 INFO MainThread:30984 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.18.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-10-31 22:54:37,817 INFO MainThread:30984 [wandb_init.py:init():804] starting run threads in backend +2024-10-31 22:54:37,917 INFO MainThread:30984 [wandb_run.py:_console_start():2241] atexit reg +2024-10-31 22:54:37,917 INFO MainThread:30984 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-10-31 22:54:37,917 INFO MainThread:30984 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-10-31 22:54:37,917 INFO MainThread:30984 [wandb_run.py:_redirect():2186] Redirects installed. +2024-10-31 22:54:37,918 INFO MainThread:30984 [wandb_init.py:init():847] run started, returning control to user process +2024-10-31 22:56:44,777 INFO MainThread:30984 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'LlamaForCausalLM', 'activation_function': 'silu', 'hidden_size': 3072, 'model_type': 'llama', 'max_position_embeddings': 4096, 'num_attention_heads': 24, 'num_hidden_layers': 28} +2024-10-31 22:56:44,777 INFO MainThread:30984 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-11-01 11:13:11,496 WARNING MsgRouterThr:30984 [router.py:message_loop():77] message_loop has been closed